data-syncmaster 0.1.1__py3-none-any.whl

syncmaster/backend/handler.py

@@ -0,0 +1,203 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+import logging
+
+from fastapi import HTTPException, Request, status
+from fastapi.responses import JSONResponse
+
+from syncmaster.exceptions import ActionNotAllowedError, SyncmasterError
+from syncmaster.exceptions.connection import (
+    ConnectionDeleteError,
+    ConnectionNotFoundError,
+    ConnectionOwnerError,
+    DuplicatedConnectionNameError,
+)
+from syncmaster.exceptions.credentials import AuthDataNotFoundError
+from syncmaster.exceptions.group import (
+    AlreadyIsGroupMemberError,
+    AlreadyIsNotGroupMemberError,
+    GroupAdminNotFoundError,
+    GroupAlreadyExistsError,
+    GroupNotFoundError,
+)
+from syncmaster.exceptions.queue import (
+    DifferentTransferAndQueueGroupError,
+    QueueDeleteError,
+    QueueNotFoundError,
+)
+from syncmaster.exceptions.run import (
+    CannotConnectToTaskQueueError,
+    CannotStopRunError,
+    RunNotFoundError,
+)
+from syncmaster.exceptions.transfer import (
+    DifferentTransferAndConnectionsGroupsError,
+    DifferentTypeConnectionsAndParamsError,
+    DuplicatedTransferNameError,
+    TransferNotFoundError,
+    TransferOwnerError,
+)
+from syncmaster.exceptions.user import UsernameAlreadyExistsError, UserNotFoundError
+from syncmaster.schemas.v1.status import StatusResponseSchema
+
+logger = logging.getLogger(__name__)
+
+
+async def http_exception_handler(request: Request, exc: HTTPException):
+    return exception_json_response(status_code=exc.status_code, detail=exc.detail)
+
+
+async def syncmsater_exception_handler(request: Request, exc: SyncmasterError):
+    if isinstance(exc, AuthDataNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Credentials not found. {exc.message}",
+        )
+
+    if isinstance(exc, ConnectionDeleteError):
+        return exception_json_response(status_code=status.HTTP_409_CONFLICT, detail=exc.message)
+
+    if isinstance(exc, ActionNotAllowedError):
+        return exception_json_response(status_code=status.HTTP_403_FORBIDDEN, detail="You have no power here")
+
+    if isinstance(exc, GroupNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Group not found",
+        )
+
+    if isinstance(exc, RunNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Run not found",
+        )
+
+    if isinstance(exc, QueueNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Queue not found",
+        )
+
+    if isinstance(exc, GroupAdminNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Admin not found",
+        )
+    if isinstance(exc, GroupAlreadyExistsError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Group name already taken",
+        )
+
+    if isinstance(exc, AlreadyIsNotGroupMemberError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="User already is not group member",
+        )
+
+    if isinstance(exc, AlreadyIsGroupMemberError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="User already is group member",
+        )
+
+    if isinstance(exc, UserNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="User not found",
+        )
+
+    if isinstance(exc, UsernameAlreadyExistsError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Username is already taken",
+        )
+
+    if isinstance(exc, ConnectionNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Connection not found",
+        )
+
+    if isinstance(exc, ConnectionOwnerError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Cannot create connection with that user_id and group_id values",
+        )
+
+    if isinstance(exc, TransferNotFoundError):
+        return exception_json_response(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Transfer not found",
+        )
+
+    if isinstance(exc, TransferOwnerError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Cannot create transfer with that group_id value",
+        )
+
+    if isinstance(exc, DifferentTransferAndConnectionsGroupsError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Connections should belong to the transfer group",
+        )
+
+    if isinstance(exc, DifferentTransferAndQueueGroupError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Queue should belong to the transfer group",
+        )
+
+    if isinstance(exc, DifferentTypeConnectionsAndParamsError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=exc.message,
+        )
+
+    if isinstance(exc, QueueDeleteError):
+        return exception_json_response(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=exc.message,
+        )
+
+    if isinstance(exc, DuplicatedConnectionNameError):
+        return exception_json_response(
+            status_code=status.HTTP_409_CONFLICT,
+            detail="The connection name already exists in the target group, please specify a new one",
+        )
+
+    if isinstance(exc, DuplicatedTransferNameError):
+        return exception_json_response(
+            status_code=status.HTTP_409_CONFLICT,
+            detail="The transfer name already exists in the target group, please specify a new one",
+        )
+
+    if isinstance(exc, CannotConnectToTaskQueueError):
+        return exception_json_response(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail=f"Syncmaster not connected to task queue. Run {exc.run_id} was failed",
+        )
+
+    if isinstance(exc, CannotStopRunError):
+        return exception_json_response(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Cannot stop run {exc.run_id}. Current status is {exc.current_status}",
+        )
+
+    logger.exception("Got unhandled error")
+    return exception_json_response(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        detail="Got unhandled exception. See logs",
+    )
+
+
+def exception_json_response(status_code: int, detail: str) -> JSONResponse:
+    return JSONResponse(
+        status_code=status_code,
+        content=StatusResponseSchema(
+            ok=False,
+            status_code=status_code,
+            message=detail,
+        ).dict(),
+    )

syncmaster/backend/logger.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0

syncmaster/backend/main.py

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from starlette.middleware.cors import CORSMiddleware
+
+from syncmaster.backend.api.deps import (
+    AuthMarker,
+    DatabaseEngineMarker,
+    DatabaseSessionMarker,
+    SettingsMarker,
+    UnitOfWorkMarker,
+)
+from syncmaster.backend.api.router import api_router
+from syncmaster.backend.handler import (
+    http_exception_handler,
+    syncmsater_exception_handler,
+)
+from syncmaster.backend.services import get_auth_scheme
+from syncmaster.config import Settings
+from syncmaster.db.factory import create_engine, create_session_factory, get_uow
+from syncmaster.exceptions import SyncmasterError
+
+
+def get_application(settings: Settings) -> FastAPI:
+    application = FastAPI(
+        title=settings.PROJECT_NAME,
+        debug=settings.DEBUG,
+    )
+    application.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    application.include_router(api_router)
+    application.exception_handler(HTTPException)(http_exception_handler)
+    application.exception_handler(SyncmasterError)(syncmsater_exception_handler)
+
+    engine = create_engine(connection_uri=settings.build_db_connection_uri())
+    session_factory = create_session_factory(engine=engine)
+
+    auth_scheme = get_auth_scheme(settings)
+
+    application.dependency_overrides.update(
+        {
+            SettingsMarker: lambda: settings,
+            DatabaseEngineMarker: lambda: engine,
+            DatabaseSessionMarker: lambda: session_factory,
+            UnitOfWorkMarker: get_uow(session_factory, settings),
+            AuthMarker: auth_scheme,
+        }
+    )
+
+    return application
+
+
+if __name__ == "__main__":
+    settings = Settings()
+    app = get_application(settings=settings)
+    uvicorn.run(app, host="0.0.0.0", port=8000)

syncmaster/backend/pre_start.py

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+import logging
+from time import sleep
+
+from syncmaster.config import Settings, TestSettings
+from tests.test_integration.test_run_transfer.conftest import get_spark_session
+
+TIMEOUT = 5
+COUNT = 12
+
+logger = logging.getLogger(__name__)
+
+
+def check_test_postgres(settings: Settings, test_settings: TestSettings) -> None:
+    from onetl.connection import Postgres
+
+    spark_session = get_spark_session(settings)
+
+    count = COUNT
+    connection = Postgres(
+        host=test_settings.TEST_POSTGRES_HOST,
+        port=test_settings.TEST_POSTGRES_PORT,
+        user=test_settings.TEST_POSTGRES_USER,
+        password=test_settings.TEST_POSTGRES_PASSWORD,
+        database=test_settings.TEST_POSTGRES_DB,
+        spark=spark_session,
+    )
+    exception = None
+    while count > 0:
+        try:
+            connection.check()
+            return
+        except Exception:
+            count -= 1
+            logger.info("Got exception on postgres. Will try after %d sec", TIMEOUT)
+            sleep(TIMEOUT)
+    if exception:
+        raise exception
+
+
+def check_test_oracle(settings: Settings, test_settings: TestSettings) -> None:
+    from onetl.connection import Oracle
+
+    spark_session = get_spark_session(settings)
+
+    count = COUNT
+    connection = Oracle(
+        host=test_settings.TEST_ORACLE_HOST,
+        port=test_settings.TEST_ORACLE_PORT,
+        user=test_settings.TEST_ORACLE_USER,
+        password=test_settings.TEST_ORACLE_PASSWORD,
+        service_name=test_settings.TEST_ORACLE_SERVICE_NAME,
+        sid=test_settings.TEST_ORACLE_SID,
+        spark=spark_session,
+    )
+    exception = None
+    while count > 0:
+        try:
+            connection.check()
+            return
+        except Exception:
+            count -= 1
+            logger.info("Got exception on oracle. Will try after %d sec", TIMEOUT)
+            sleep(TIMEOUT)
+    if exception:
+        raise exception
+
+
+def check_test_hive(settings: Settings, test_settings: TestSettings) -> None:
+    from onetl.connection import Hive
+
+    spark_session = get_spark_session(settings)
+
+    count = COUNT
+    connection = Hive(
+        cluster=test_settings.TEST_HIVE_CLUSTER,
+        spark=spark_session,
+    )
+    exception = None
+    while count > 0:
+        try:
+            connection.check()
+            connection.execute("SHOW DATABASES")
+            return
+        except Exception:
+            count -= 1
+            logger.info("Got exception on hive. will try after %d sec", TIMEOUT)
+            sleep(TIMEOUT)
+    if exception:
+        raise exception
+
+
+settings = Settings()

syncmaster/backend/services/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.backend.services.auth import get_auth_scheme, get_user
+from syncmaster.backend.services.unit_of_work import UnitOfWork

syncmaster/backend/services/auth.py

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from collections.abc import Awaitable, Callable
+
+from fastapi import Depends, Request, status
+from fastapi.exceptions import HTTPException
+from fastapi.security import OAuth2PasswordBearer
+
+from syncmaster.backend.api.deps import AuthMarker, SettingsMarker, UnitOfWorkMarker
+from syncmaster.backend.api.v1.auth.utils import decode_jwt
+from syncmaster.backend.services.unit_of_work import UnitOfWork
+from syncmaster.config import Settings
+from syncmaster.db.models import User
+
+
+def get_user(
+    is_active: bool = False,
+    is_superuser: bool = False,
+) -> Callable[[str, Settings, UnitOfWork], Awaitable[User]]:
+    async def wrapper(
+        token: str = Depends(AuthMarker),
+        settings: Settings = Depends(SettingsMarker),
+        unit_of_work: UnitOfWork = Depends(UnitOfWorkMarker),
+    ) -> User:
+        async with unit_of_work:
+            token_data = decode_jwt(token, settings=settings)
+            if token_data is None:
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="You are not authorized",
+                )
+            user = await unit_of_work.user.read_by_id(user_id=token_data.user_id)
+            if user is None:
+                raise HTTPException(
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail="User not found",
+                )
+            if is_active and not user.is_active:
+                raise HTTPException(
+                    status_code=status.HTTP_403_FORBIDDEN,
+                    detail="Inactive user",
+                )
+            if is_superuser and not user.is_superuser:
+                raise HTTPException(
+                    status_code=status.HTTP_403_FORBIDDEN,
+                    detail="You have no power here",
+                )
+        return user
+
+    return wrapper
+
+
+def get_auth_scheme(
+    settings: Settings,
+) -> Callable[[Request], str] | OAuth2PasswordBearer:
+    if settings.DEBUG:
+        return OAuth2PasswordBearer(tokenUrl=settings.AUTH_TOKEN_URL)
+    return lambda request: "Here will be keycloak"

syncmaster/backend/services/unit_of_work.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from syncmaster.config import Settings
+from syncmaster.db.models import AuthData
+from syncmaster.db.repositories import (
+    ConnectionRepository,
+    CredentialsRepository,
+    GroupRepository,
+    QueueRepository,
+    RunRepository,
+    TransferRepository,
+    UserRepository,
+)
+
+
+class UnitOfWork:
+    def __init__(
+        self,
+        session: AsyncSession,
+        settings: Settings,
+    ):
+        self._session = session
+        self.user = UserRepository(session=session)
+        self.group = GroupRepository(session=session)
+        self.connection = ConnectionRepository(session=session)
+        self.transfer = TransferRepository(session=session)
+        self.run = RunRepository(session=session)
+        self.credentials = CredentialsRepository(
+            session=session,
+            settings=settings,
+            model=AuthData,
+        )
+        self.queue = QueueRepository(session=session)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        if exc_type:
+            await self._session.rollback()
+        else:
+            await self._session.commit()

syncmaster/config.py

@@ -0,0 +1,110 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from enum import StrEnum
+from typing import TYPE_CHECKING
+
+from pydantic import model_validator
+from pydantic.types import ImportString
+from pydantic_settings import BaseSettings
+
+if TYPE_CHECKING:
+    pass
+
+
+class EnvTypes(StrEnum):
+    LOCAL = "LOCAL"
+
+
+class Settings(BaseSettings):
+    ENV: EnvTypes = EnvTypes.LOCAL
+    DEBUG: bool = True
+
+    PROJECT_NAME: str = "SyncMaster"
+
+    SECRET_KEY: str = "secret"
+    SECURITY_ALGORITHM: str = "HS256"
+    AUTH_TOKEN_URL: str = "v1/auth/token"
+    CRYPTO_KEY: str = "UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94="
+
+    POSTGRES_HOST: str
+    POSTGRES_PORT: int
+    POSTGRES_DB: str
+    POSTGRES_USER: str
+    POSTGRES_PASSWORD: str
+
+    RABBITMQ_HOST: str
+    RABBITMQ_PORT: int
+    RABBITMQ_USER: str
+    RABBITMQ_PASSWORD: str
+
+    def build_db_connection_uri(
+        self,
+        *,
+        driver: str | None = None,
+        host: str | None = None,
+        port: int | None = None,
+        user: str | None = None,
+        password: str | None = None,
+        database: str | None = None,
+    ) -> str:
+        return "postgresql+{}://{}:{}@{}:{}/{}".format(
+            driver or "asyncpg",
+            user or self.POSTGRES_USER,
+            password or self.POSTGRES_PASSWORD,
+            host or self.POSTGRES_HOST,
+            port or self.POSTGRES_PORT,
+            database or self.POSTGRES_DB,
+        )
+
+    def build_rabbit_connection_uri(
+        self,
+        *,
+        host: str | None = None,
+        port: int | None = None,
+        user: str | None = None,
+        password: str | None = None,
+    ) -> str:
+        return "amqp://{}:{}@{}:{}//".format(
+            user or self.RABBITMQ_USER,
+            password or self.RABBITMQ_PASSWORD,
+            host or self.RABBITMQ_HOST,
+            port or self.RABBITMQ_PORT,
+        )
+
+    TOKEN_EXPIRED_TIME: int = 60 * 60 * 10  # 10 hours
+    CREATE_SPARK_SESSION_FUNCTION: ImportString = "syncmaster.worker.spark.get_worker_spark_session"
+
+
+class TestSettings(BaseSettings):
+    TEST_POSTGRES_HOST: str
+    TEST_POSTGRES_PORT: int
+    TEST_POSTGRES_DB: str
+    TEST_POSTGRES_USER: str
+    TEST_POSTGRES_PASSWORD: str
+
+    TEST_ORACLE_HOST: str
+    TEST_ORACLE_PORT: int
+    TEST_ORACLE_USER: str
+    TEST_ORACLE_PASSWORD: str
+    TEST_ORACLE_SID: str | None = None
+    TEST_ORACLE_SERVICE_NAME: str | None = None
+
+    TEST_HIVE_CLUSTER: str
+    TEST_HIVE_USER: str
+    TEST_HIVE_PASSWORD: str
+
+    TEST_S3_HOST: str
+    TEST_S3_PORT: int
+    TEST_S3_BUCKET: str
+    TEST_S3_ACCESS_KEY: str
+    TEST_S3_SECRET_KEY: str
+    TEST_S3_PROTOCOL: str = "http"
+    TEST_S3_ADDITIONAL_PARAMS: dict = {}
+
+    @model_validator(mode="before")
+    def check_sid_and_service_name(cls, values):
+        sid = values.get("TEST_ORACLE_SID")
+        service_name = values.get("TEST_ORACLE_SERVICE_NAME")
+        if (sid is None) == (service_name is None):
+            raise ValueError("Connection must have one param: sid or service name")
+        return values

syncmaster/db/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0

syncmaster/db/alembic.ini

@@ -0,0 +1,41 @@
+[alembic]
+script_location = %(here)s/migrations
+file_template = %%(year)d-%%(month).2d-%%(day).2d_%%(rev)s_%%(slug)s
+prepend_sys_path = .
+version_path_separator = os
+
+[post_write_hooks]
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

syncmaster/db/base.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+import re
+
+from sqlalchemy import MetaData
+from sqlalchemy.orm import DeclarativeBase, declared_attr
+
+convention = {
+    "all_column_names": lambda constraint, table: "_".join([column.name for column in constraint.columns.values()]),
+    "ix": "ix__%(table_name)s__%(all_column_names)s",
+    "uq": "uq__%(table_name)s__%(all_column_names)s",
+    "ck": "ck__%(table_name)s__%(constraint_name)s",
+    "fk": ("fk__%(table_name)s__%(all_column_names)s__%(referred_table_name)s"),
+    "pk": "pk__%(table_name)s",
+}
+
+model_metadata = MetaData(naming_convention=convention)
+
+
+# as_declarative decorator causes mypy errors.
+# Use inheritance from DeclarativeBase.
+class Base(DeclarativeBase):
+    metadata = model_metadata
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        name_list = re.findall(r"[A-Z][a-z\d]*", cls.__name__)
+        return "_".join(name_list).lower()

syncmaster/db/factory.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
+# SPDX-License-Identifier: Apache-2.0
+from collections.abc import AsyncGenerator, Callable
+from typing import Any
+
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from syncmaster.backend.services import UnitOfWork
+from syncmaster.config import Settings
+
+
+def create_engine(connection_uri: str, **engine_kwargs: Any) -> AsyncEngine:
+    return create_async_engine(url=connection_uri, **engine_kwargs)
+
+
+def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
+    return async_sessionmaker(
+        bind=engine,
+        class_=AsyncSession,
+        expire_on_commit=False,
+    )
+
+
+def get_uow(
+    session_factory: async_sessionmaker[AsyncSession],
+    settings: Settings,
+) -> Callable[[], AsyncGenerator[UnitOfWork, None]]:
+    async def wrapper():
+        async with session_factory() as session:
+            yield UnitOfWork(session=session, settings=settings)
+
+    return wrapper

syncmaster/db/migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration with an async dbapi.