data-syncmaster 0.1.1__py3-none-any.whl

syncmaster/schemas/v1/connections/oracle.py

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, Field, SecretStr, model_validator
+
+from syncmaster.schemas.v1.connection_types import ORACLE_TYPE
+from syncmaster.schemas.v1.types import NameConstr
+
+
+class OracleBaseSchema(BaseModel):
+    type: ORACLE_TYPE
+
+    class Config:
+        from_attributes = True
+
+
+class ReadOracleConnectionSchema(OracleBaseSchema):
+    host: str
+    port: int
+    service_name: str | None = None
+    sid: str | None = None
+    additional_params: dict = Field(default_factory=dict)
+
+
+class ReadOracleAuthSchema(OracleBaseSchema):
+    user: str
+
+
+class UpdateOracleConnectionSchema(OracleBaseSchema):
+    host: str | None = None
+    sid: str | None = None
+    service_name: str | None = None
+    additional_params: dict | None = Field(default_factory=dict)
+
+
+class UpdateOracleAuthSchema(OracleBaseSchema):
+    user: NameConstr | None = None  # noqa: F722
+    password: SecretStr | None = None
+
+
+class CreateOracleConnectionSchema(OracleBaseSchema):
+    host: str
+    port: int
+    service_name: str | None = None
+    sid: str | None = None
+    additional_params: dict = Field(default_factory=dict)
+
+    @model_validator(mode="before")
+    def check_owner_id(cls, values):
+        sid, service_name = values.get("sid"), values.get("service_name")
+        if sid and service_name:
+            raise ValueError("You must specify either sid or service_name but not both")
+        return values
+
+
+class CreateOracleAuthSchema(OracleBaseSchema):
+    user: str
+    password: SecretStr

syncmaster/schemas/v1/connections/postgres.py

@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, Field, SecretStr
+
+from syncmaster.schemas.v1.connection_types import POSTGRES_TYPE
+
+
+class PostgresBaseSchema(BaseModel):
+    type: POSTGRES_TYPE
+
+    class Config:
+        from_attributes = True
+
+
+class ReadPostgresConnectionSchema(PostgresBaseSchema):
+    host: str
+    port: int = Field(gt=0, le=65535)
+    database_name: str
+    additional_params: dict = Field(default_factory=dict)
+
+
+class ReadPostgresAuthSchema(PostgresBaseSchema):
+    user: str
+
+
+class UpdatePostgresConnectionSchema(PostgresBaseSchema):
+    host: str | None = None
+    port: int | None = None
+    database_name: str | None = None
+    additional_params: dict | None = Field(default_factory=dict)
+
+
+class UpdatePostgresAuthSchema(PostgresBaseSchema):
+    user: str | None = None
+    password: SecretStr | None = None
+
+
+class CreatePostgresConnectionSchema(PostgresBaseSchema):
+    host: str
+    port: int
+    database_name: str
+    additional_params: dict = Field(default_factory=dict)
+
+
+class CreatePostgresAuthSchema(PostgresBaseSchema):
+    user: str
+    password: SecretStr

syncmaster/schemas/v1/connections/s3.py

@@ -0,0 +1,66 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from typing import Literal
+
+from pydantic import BaseModel, SecretStr, model_validator
+
+from syncmaster.schemas.v1.connection_types import S3_TYPE
+
+
+class S3BaseSchema(BaseModel):
+    type: S3_TYPE
+
+    class Config:
+        from_attributes = True
+
+
+class S3ReadConnectionSchema(S3BaseSchema):
+    host: str
+    bucket: str
+    port: int | None = None
+    region: str | None = None
+    protocol: Literal["http", "https"] = "https"
+    bucket_style: Literal["domain", "path"] = "domain"
+
+
+class S3CreateConnectionSchema(S3BaseSchema):
+    host: str
+    bucket: str
+    port: int | None = None
+    region: str | None = None
+    protocol: Literal["http", "https"] = "https"
+    bucket_style: Literal["domain", "path"] = "domain"
+
+    @model_validator(mode="before")
+    def validate_port(cls, values: dict) -> dict:
+        port = values.get("port")
+        protocol = values.get("protocol")
+
+        if port is None:
+            port = 443 if protocol == "https" else 80
+            values["port"] = port
+
+        return values
+
+
+class S3UpdateConnectionSchema(S3BaseSchema):
+    host: str | None = None
+    bucket: str | None = None
+    port: int | None = None
+    region: str | None = None
+    protocol: Literal["http", "https"] | None = None
+    bucket_style: Literal["domain", "path"] | None = None
+
+
+class S3CreateAuthSchema(S3BaseSchema):
+    access_key: str
+    secret_key: SecretStr
+
+
+class S3ReadAuthSchema(S3BaseSchema):
+    access_key: str
+
+
+class S3UpdateAuthSchema(S3BaseSchema):
+    access_key: str | None = None
+    secret_key: SecretStr | None = None

syncmaster/schemas/v1/file_formats.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from typing import Literal
+
+CSV_FORMAT = Literal["csv"]
+JSONLINE_FORMAT = Literal["jsonline"]
+JSON_FORMAT = Literal["json"]

syncmaster/schemas/v1/groups.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from pydantic import BaseModel
+
+from syncmaster.db.models import GroupMemberRole
+from syncmaster.schemas.v1.page import PageSchema
+from syncmaster.schemas.v1.types import NameConstr
+
+
+class UpdateGroupSchema(BaseModel):
+    name: NameConstr  # noqa: F722
+    description: str
+    owner_id: int
+
+
+class CreateGroupSchema(BaseModel):
+    name: NameConstr  # noqa: F722
+    description: str
+
+
+class AddUserSchema(BaseModel):
+    role: GroupMemberRole
+
+    class Config:
+        from_attributes = True
+
+
+class ReadGroupSchema(BaseModel):
+    id: int
+    name: NameConstr  # noqa: F722
+    description: str
+    owner_id: int
+
+    class Config:
+        from_attributes = True
+
+
+class GroupPageSchema(PageSchema):
+    items: list[ReadGroupSchema]

syncmaster/schemas/v1/page.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+import abc
+from typing import Any
+
+from pydantic import BaseModel
+
+from syncmaster.db.utils import Pagination
+
+
+class MetaPageSchema(BaseModel):
+    page: int
+    pages: int
+    total: int
+    page_size: int
+    has_next: bool
+    has_previous: bool
+    next_page: int | None = None
+    previous_page: int | None = None
+
+
+class PageSchema(BaseModel, abc.ABC):
+    meta: MetaPageSchema
+    items: list[Any]
+
+    @classmethod
+    def from_pagination(cls, pagination: Pagination):
+        return cls(
+            meta=MetaPageSchema(
+                page=pagination.page,
+                pages=pagination.pages,
+                page_size=pagination.page_size,
+                total=pagination.total,
+                has_next=pagination.has_next,
+                has_previous=pagination.has_previous,
+                next_page=pagination.next_page,
+                previous_page=pagination.previous_page,
+            ),
+            items=pagination.items,
+        )

syncmaster/schemas/v1/queue.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from pydantic import BaseModel, Field, constr
+
+from syncmaster.schemas.v1.page import PageSchema
+
+
+class CreateQueueSchema(BaseModel):
+    name: constr(max_length=128, pattern=r"^[-_a-zA-Z0-9]+$") = Field(  # noqa: F722
+        ...,
+        description="Queue name",
+    )
+    group_id: int = Field(..., description="Queue owner group id")
+    description: str = Field(default="", description="Additional description")
+
+
+class ReadQueueSchema(BaseModel):
+    name: str
+    description: str | None = None
+    group_id: int
+    id: int
+
+    class Config:
+        from_attributes = True
+
+
+class QueuePageSchema(PageSchema):
+    items: list[ReadQueueSchema]
+
+
+class UpdateQueueSchema(BaseModel):
+    description: str | None = None

syncmaster/schemas/v1/status.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel
+
+
+class StatusResponseSchema(BaseModel):
+    ok: bool
+    status_code: int
+    message: str
+
+
+class StatusCopyTransferResponseSchema(StatusResponseSchema):
+    source_connection_id: int
+    target_connection_id: int
+    copied_transfer_id: int

syncmaster/schemas/v1/transfer_types.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from typing import Literal
+
+FULL_TYPE = Literal["full"]
+INCREMENTAL_TYPE = Literal["incremental"]

syncmaster/schemas/v1/transfers/__init__.py

@@ -0,0 +1,172 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, model_validator
+
+from syncmaster.schemas.v1.connections.connection import ReadConnectionSchema
+from syncmaster.schemas.v1.page import PageSchema
+from syncmaster.schemas.v1.transfers.db import (
+    HiveReadTransferSourceAndTarget,
+    OracleReadTransferSourceAndTarget,
+    PostgresReadTransferSourceAndTarget,
+)
+from syncmaster.schemas.v1.transfers.file.hdfs import (
+    HDFSCreateTransferSource,
+    HDFSCreateTransferTarget,
+    HDFSReadTransferSource,
+    HDFSReadTransferTarget,
+)
+from syncmaster.schemas.v1.transfers.file.s3 import (
+    S3CreateTransferSource,
+    S3CreateTransferTarget,
+    S3ReadTransferSource,
+    S3ReadTransferTarget,
+)
+from syncmaster.schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy
+from syncmaster.schemas.v1.types import NameConstr
+
+ReadTransferSchemaSource = (
+    PostgresReadTransferSourceAndTarget
+    | HDFSReadTransferSource
+    | HiveReadTransferSourceAndTarget
+    | OracleReadTransferSourceAndTarget
+    | S3ReadTransferSource
+)
+
+ReadTransferSchemaTarget = (
+    PostgresReadTransferSourceAndTarget
+    | HDFSReadTransferTarget
+    | HiveReadTransferSourceAndTarget
+    | OracleReadTransferSourceAndTarget
+    | S3ReadTransferTarget
+)
+
+CreateTransferSchemaSource = (
+    PostgresReadTransferSourceAndTarget
+    | HDFSCreateTransferSource
+    | HiveReadTransferSourceAndTarget
+    | OracleReadTransferSourceAndTarget
+    | S3CreateTransferSource
+)
+
+CreateTransferSchemaTarget = (
+    PostgresReadTransferSourceAndTarget
+    | HDFSCreateTransferTarget
+    | HiveReadTransferSourceAndTarget
+    | OracleReadTransferSourceAndTarget
+    | S3CreateTransferTarget
+)
+
+UpdateTransferSchemaSource = (
+    PostgresReadTransferSourceAndTarget
+    | HDFSReadTransferSource
+    | HiveReadTransferSourceAndTarget
+    | OracleReadTransferSourceAndTarget
+    | S3CreateTransferSource
+    | None
+)
+
+UpdateTransferSchemaTarget = (
+    PostgresReadTransferSourceAndTarget
+    | HDFSReadTransferSource
+    | HiveReadTransferSourceAndTarget
+    | OracleReadTransferSourceAndTarget
+    | S3CreateTransferTarget
+    | None
+)
+
+
+class CopyTransferSchema(BaseModel):
+    new_group_id: int
+    new_queue_id: int
+    new_source_connection_name: NameConstr | None = None  # noqa: F722
+    new_target_connection_name: NameConstr | None = None  # noqa: F722
+    new_name: NameConstr | None = None  # noqa: F722
+    remove_source: bool = False
+
+
+class ReadTransferSchema(BaseModel):
+    id: int
+    group_id: int
+    source_connection_id: int
+    target_connection_id: int
+    name: str
+    description: str
+    is_scheduled: bool
+    schedule: str
+    queue_id: int
+    source_params: ReadTransferSchemaSource = Field(
+        ...,
+        discriminator="type",
+    )
+    target_params: ReadTransferSchemaTarget = Field(
+        ...,
+        discriminator="type",
+    )
+    strategy_params: FullStrategy | IncrementalStrategy = Field(
+        ...,
+        discriminator="type",
+    )
+
+    class Config:
+        from_attributes = True
+
+
+class CreateTransferSchema(BaseModel):
+    group_id: int = Field(..., description="Transfer owner group id")
+    source_connection_id: int = Field(..., description="id of the connection that will be the data source")
+    target_connection_id: int = Field(..., description="id of the connection that will be the data receiver")
+    name: NameConstr = Field(..., description="Transfer name")  # noqa: F722
+    description: str = Field(..., description="Additional description")
+    is_scheduled: bool = Field(..., description="Is the transfer on schedule")
+    queue_id: int = Field(..., description="id of the queue in which the transfer will be performed")
+    schedule: str | None = Field(None, description="Execution schedule in cron format")
+    source_params: CreateTransferSchemaSource = Field(
+        ...,
+        discriminator="type",
+        description="Data source parameters",
+    )
+    target_params: CreateTransferSchemaTarget = Field(
+        ...,
+        discriminator="type",
+        description="Data receiver parameters",
+    )
+    strategy_params: FullStrategy | IncrementalStrategy = Field(
+        ...,
+        discriminator="type",
+        description="Incremental or archive download options",
+    )
+
+    @model_validator(mode="before")
+    def check_owner_id(cls, values):
+        is_scheduled, schedule = values.get("is_scheduled"), values.get("schedule")
+        if is_scheduled and schedule is None:
+            # TODO make checking cron string
+            raise ValueError("If transfer must be scheduled than set schedule param")
+        return values
+
+
+class UpdateTransferSchema(BaseModel):
+    source_connection_id: int | None = None
+    target_connection_id: int | None = None
+    name: NameConstr | None = None  # noqa: F722
+    description: str | None = None
+    is_scheduled: bool | None = None
+    schedule: str | None = None
+    new_queue_id: int | None = None
+    source_params: UpdateTransferSchemaSource = Field(discriminator="type", default=None)
+    target_params: UpdateTransferSchemaTarget = Field(discriminator="type", default=None)
+    strategy_params: FullStrategy | IncrementalStrategy | None = Field(discriminator="type", default=None)
+
+
+class ReadFullTransferSchema(ReadTransferSchema):
+    source_connection: ReadConnectionSchema
+    target_connection: ReadConnectionSchema
+
+    class Config:
+        from_attributes = True
+
+
+class TransferPageSchema(PageSchema):
+    items: list[ReadTransferSchema]

syncmaster/schemas/v1/transfers/db.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+from syncmaster.schemas.v1.connection_types import HIVE_TYPE, ORACLE_TYPE, POSTGRES_TYPE
+
+
+class ReadDBTransfer(BaseModel):
+    table_name: str
+
+
+class HiveReadTransferSourceAndTarget(ReadDBTransfer):
+    type: HIVE_TYPE
+
+
+class OracleReadTransferSourceAndTarget(ReadDBTransfer):
+    type: ORACLE_TYPE
+
+
+class PostgresReadTransferSourceAndTarget(ReadDBTransfer):
+    type: POSTGRES_TYPE

syncmaster/schemas/v1/transfers/file/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0

syncmaster/schemas/v1/transfers/file/base.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+
+from pydantic import BaseModel, Field, validator
+
+from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine
+
+
+def validate_directory_path(path: str) -> PurePosixPath:
+    return PurePosixPath(path)
+
+
+# At the moment the ReadTransferSourceParams and ReadTransferTargetParams
+# classes are identical but may change in the future
+class ReadFileTransferSource(BaseModel):
+    directory_path: str
+    file_format: CSV | JSONLine | JSON = Field(..., discriminator="type")
+
+
+class ReadFileTransferTarget(BaseModel):
+    directory_path: str
+    file_format: CSV | JSONLine = Field(..., discriminator="type")  # JSON format is not supported for writing
+
+
+# At the moment the CreateTransferSourceParams and CreateTransferTargetParams
+# classes are identical but may change in the future
+class CreateFileTransferSource(BaseModel):
+    directory_path: PurePosixPath
+    file_format: CSV | JSONLine | JSON = Field(..., discriminator="type")
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    _validate_dir_path = validator("directory_path", allow_reuse=True, pre=True)(validate_directory_path)
+
+
+class CreateFileTransferTarget(BaseModel):
+    directory_path: PurePosixPath
+    file_format: CSV | JSONLine = Field(..., discriminator="type")  # JSON FORMAT IS NOT SUPPORTED AS A TARGET !
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    _validate_dir_path = validator("directory_path", allow_reuse=True, pre=True)(validate_directory_path)

syncmaster/schemas/v1/transfers/file/hdfs.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from syncmaster.schemas.v1.connection_types import HDFS_TYPE
+from syncmaster.schemas.v1.transfers.file.base import (
+    CreateFileTransferSource,
+    CreateFileTransferTarget,
+    ReadFileTransferSource,
+    ReadFileTransferTarget,
+)
+
+
+class HDFSReadTransferSource(ReadFileTransferSource):
+    type: HDFS_TYPE
+
+
+class HDFSReadTransferTarget(ReadFileTransferTarget):
+    type: HDFS_TYPE
+
+
+class HDFSCreateTransferSource(CreateFileTransferSource):
+    type: HDFS_TYPE
+
+
+class HDFSCreateTransferTarget(CreateFileTransferTarget):
+    type: HDFS_TYPE

syncmaster/schemas/v1/transfers/file/s3.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from syncmaster.schemas.v1.connection_types import S3_TYPE
+from syncmaster.schemas.v1.transfers.file.base import (
+    CreateFileTransferSource,
+    CreateFileTransferTarget,
+    ReadFileTransferSource,
+    ReadFileTransferTarget,
+)
+
+
+class S3ReadTransferSource(ReadFileTransferSource):
+    type: S3_TYPE
+
+
+class S3ReadTransferTarget(ReadFileTransferTarget):
+    type: S3_TYPE
+
+
+class S3CreateTransferSource(CreateFileTransferSource):
+    type: S3_TYPE
+
+
+class S3CreateTransferTarget(CreateFileTransferTarget):
+    type: S3_TYPE

syncmaster/schemas/v1/transfers/file_format.py

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+from syncmaster.schemas.v1.file_formats import CSV_FORMAT, JSON_FORMAT, JSONLINE_FORMAT
+
+
+class CSV(BaseModel):
+    type: CSV_FORMAT
+    delimiter: str = ","
+    encoding: str = "utf-8"
+    quote: str = '"'
+    escape: str = "\\"
+    header: bool = False
+    line_sep: str = "\n"
+
+
+class JSONLine(BaseModel):
+    type: JSONLINE_FORMAT
+    encoding: str = "utf-8"
+    line_sep: str = "\n"
+
+
+class JSON(BaseModel):
+    type: JSON_FORMAT
+    encoding: str = "utf-8"
+    line_sep: str = "\n"

syncmaster/schemas/v1/transfers/run.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel
+
+from syncmaster.db.models import Status
+from syncmaster.schemas.v1.page import PageSchema
+
+
+class ShortRunSchema(BaseModel):
+    id: int
+    transfer_id: int
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    status: Status
+    log_url: str | None = None
+
+    class Config:
+        from_attributes = True
+
+
+class RunPageSchema(PageSchema):
+    items: list[ShortRunSchema]
+
+
+class ReadRunSchema(ShortRunSchema):
+    transfer_dump: dict
+
+    class Config:
+        from_attributes = True
+
+
+class CreateRunSchema(BaseModel):
+    transfer_id: int