data-syncmaster 0.2.2__tar.gz → 0.2.4__tar.gz

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: data-syncmaster
-Version: 0.2.2
+Version: 0.2.4
 Summary: Syncmaster REST API + Worker
 License: Apache-2.0
 Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
@@ -35,11 +35,12 @@ Requires-Dist: fastapi (>=0.115.12,<0.116.0) ; extra == "server"
 Requires-Dist: horizon-hwm-store (>=1.1.2,<2.0.0) ; extra == "worker"
 Requires-Dist: itsdangerous (>=2.2.0,<3.0.0) ; extra == "server"
 Requires-Dist: jinja2 (>=3.1.6,<4.0.0) ; extra == "server" or extra == "worker"
-Requires-Dist: onetl[all] (>=0.13.4,<0.14.0) ; extra == "worker"
+Requires-Dist: onetl[all] (>=0.13.5,<0.14.0) ; extra == "worker"
 Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0) ; extra == "server" or extra == "worker"
-Requires-Dist: pydantic (>=2.11.1,<3.0.0)
+Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: pyjwt (>=2.10.1,<3.0.0) ; extra == "server"
+Requires-Dist: pyspark (<4.0.0) ; extra == "worker"
 Requires-Dist: python-json-logger ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: python-keycloak (>=5.3.1,<6.0.0) ; extra == "server"
 Requires-Dist: python-multipart (>=0.0.20,<0.0.21) ; extra == "server"
@@ -47,8 +48,8 @@ Requires-Dist: pyyaml ; extra == "server" or extra == "worker" or extra == "sche
 Requires-Dist: sqlalchemy (>=2.0.40,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: sqlalchemy-utils (>=0.41.2,<0.42.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: starlette-exporter (>=0.23.0,<0.24.0) ; extra == "server"
-Requires-Dist: uuid6 (>=2024.7.10,<2025.0.0) ; extra == "server" or extra == "worker"
-Requires-Dist: uvicorn (>=0.34.0,<0.35.0) ; extra == "server"
+Requires-Dist: uuid6 (>=2025.0.0,<2026.0.0) ; extra == "server" or extra == "worker"
+Requires-Dist: uvicorn (>=0.34,<0.36) ; extra == "server"
 Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
 Project-URL: Documentation, https://syncmaster.readthedocs.io
 Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "data-syncmaster"
-version = "0.2.2"
+version = "0.2.4"
 license = "Apache-2.0"
 description = "Syncmaster REST API + Worker"
 authors = ["DataOps.ETL <onetools@mts.ru>"]
@@ -44,22 +44,23 @@ exclude = [
 
 [tool.poetry.dependencies]
 python = "^3.11"
-pydantic = "^2.11.1"
+pydantic = "^2.11.7"
 pydantic-settings = { version = "^2.8.1", optional = true }
 sqlalchemy = { version = "^2.0.40", optional = true }
 sqlalchemy-utils = { version = "^0.41.2", optional = true }
 fastapi = { version = "^0.115.12", optional = true}
 asgi-correlation-id = {version = "^4.3.4", optional = true}
-uvicorn = { version = "^0.34.0", optional = true }
+uvicorn = { version = ">=0.34,<0.36", optional = true }
 alembic = { version = "^1.15.2", optional = true }
 pyjwt = { version = "^2.10.1", optional = true }
 jinja2 = { version = "^3.1.6", optional = true }
 python-multipart = { version = "^0.0.20", optional = true }
 celery = { version = "^5.5.0", optional = true }
-onetl = { version = "^0.13.4", extras = ["all"], optional = true }
+onetl = { version = "^0.13.5", extras = ["all"], optional = true }
+pyspark = { version = "<4.0.0", optional = true }
 pyyaml = { version = "*", optional = true }
 psycopg2-binary = { version = "^2.9.10", optional = true }
-uuid6 = {version = "^2024.7.10", optional = true}
+uuid6 = {version = "^2025.0.0", optional = true}
 coloredlogs = {version = "*", optional = true}
 python-json-logger = {version = "*", optional = true}
 asyncpg = { version = "^0.30.0", optional = true }
@@ -99,6 +100,7 @@ worker = [
     "sqlalchemy-utils",
     "celery",
     "onetl",
+    "pyspark",
     "asgi-correlation-id",
     "jinja2",
     "psycopg2-binary",
@@ -124,15 +126,15 @@ scheduler = [
 [tool.poetry.group.test.dependencies]
 pytest = "^8.3.5"
 httpx = "^0.28.1"
-pytest-asyncio = "^0.26.0"
+pytest-asyncio = ">=0.26.0,<1.0"
 pytest-randomly = "^3.15.0"
 pytest-rerunfailures = "^15.0"
 pytest-deadfixtures = "^2.2.1"
 pytest-mock = "^3.14.0"
 pytest-lazy-fixtures = "^1.1.1"
-faker = "^37.1.0"
-coverage = "^7.8.0"
-gevent = "^24.11.1"
+faker = "^37.4.0"
+coverage = "^7.9.1"
+gevent = ">=24.11.1,<26.0.0"
 responses = "^0.25.7"
 
 [tool.poetry.group.dev.dependencies]

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/__init__.py

@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 
-_raw_version = "0.2.2"
+_raw_version = "0.2.4"
 # version always contain only release number like 0.0.1
 __version__ = ".".join(_raw_version.split(".")[:3])  # noqa: WPS410

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/server/api/v1/runs.py

@@ -23,7 +23,6 @@ from syncmaster.schemas.v1.transfers.run import (
 from syncmaster.server.dependencies import Stub
 from syncmaster.server.services.get_user import get_user
 from syncmaster.server.services.unit_of_work import UnitOfWork
-from syncmaster.server.settings import ServerAppSettings as Settings
 
 router = APIRouter(tags=["Runs"], responses=get_error_responses())
 
@@ -82,7 +81,6 @@ async def read_run(
 @router.post("/runs")
 async def start_run(  # noqa: WPS217
     create_run_data: CreateRunSchema,
-    settings: Annotated[Settings, Depends(Stub(Settings))],
     celery: Annotated[Celery, Depends(Stub(Celery))],
     unit_of_work: UnitOfWork = Depends(UnitOfWork),
     current_user: User = Depends(get_user(is_active=True)),

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/server/middlewares/monitoring/metrics.py

@@ -37,7 +37,7 @@ def apply_monitoring_metrics_middleware(app: FastAPI, settings: MonitoringSettin
     app.add_middleware(
         PrometheusMiddleware,
         app_name=slugify(app.title),
-        skip_paths=skip_paths,
+        skip_paths=sorted(skip_paths),
         **settings.model_dump(exclude={"enabled", "skip_paths"}),
     )
     app.include_router(router)

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/server/middlewares/session.py

@@ -10,8 +10,8 @@ from syncmaster.server.settings.server.session import SessionSettings
 def apply_session_middleware(app: FastAPI, settings: SessionSettings) -> FastAPI:
     """Add SessionMiddleware middleware to the application."""
 
-    app.add_middleware(
-        SessionMiddleware,
-        **settings.model_dump(),
-    )
+    settings_dict = settings.model_dump(exclude={"secret_key"})
+    settings_dict["secret_key"] = settings.secret_key.get_secret_value()
+
+    app.add_middleware(SessionMiddleware, **settings_dict)
     return app

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/server/settings/auth/jwt.py

@@ -23,7 +23,11 @@ class JWTSettings(BaseModel):
             """
             Secret key for signing JWT tokens.
 
-            Can be any string. It is recommended to generate random value for every application instance.
+            Can be any string. It is recommended to generate random value for every application instance, e.g.:
+
+            .. code:: shell
+
+                pwgen 32 1
             """,
         ),
     )

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/server/settings/server/session.py

@@ -2,7 +2,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-from pydantic import BaseModel, ConfigDict, Field
+import textwrap
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
 
 
 class SessionSettings(BaseModel):
@@ -38,8 +40,23 @@ class SessionSettings(BaseModel):
 
     """
 
-    secret_key: str = Field(description="A random string for signing cookies.")
-    session_cookie: str | None = Field(default="session", description="Name of the session cookie.")
+    secret_key: SecretStr = Field(
+        description=textwrap.dedent(
+            """
+            Secret key for encrypting cookies.
+
+            Can be any string. It is recommended to generate random value for every application instance, e.g.:
+
+            .. code:: shell
+
+                pwgen 32 1
+            """,
+        ),
+    )
+    session_cookie: str | None = Field(
+        default="session",
+        description="Name of the session cookie. Change this if there are multiple application under the same domain.",
+    )
     max_age: int | None = Field(
         default=14 * 24 * 60 * 60,
         description="Session expiry time in seconds. Defaults to 2 weeks.",

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/settings/log/colored.yml

@@ -40,7 +40,7 @@ loggers:
     level: INFO
     propagate: false
   celery:
-    level: DEBUG
+    level: INFO
     handlers: [celery]
     propagate: false
   scheduler:
@@ -51,3 +51,7 @@ loggers:
     handlers: [main]
     level: WARNING
     propagate: false
+  hdfs.client:
+    handlers: [main]
+    level: WARNING
+    propagate: false

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/settings/log/json.yml

@@ -39,7 +39,7 @@ loggers:
     level: INFO
     propagate: false
   celery:
-    level: DEBUG
+    level: INFO
     handlers: [celery]
     propagate: false
   scheduler:
@@ -50,3 +50,7 @@ loggers:
     handlers: [main]
     level: WARNING
     propagate: false
+  hdfs.client:
+    handlers: [main]
+    level: WARNING
+    propagate: false

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/settings/log/plain.yml

@@ -40,7 +40,7 @@ loggers:
     level: INFO
     propagate: false
   celery:
-    level: DEBUG
+    level: INFO
     handlers: [celery]
     propagate: false
   scheduler:
@@ -51,3 +51,7 @@ loggers:
     handlers: [main]
     level: WARNING
     propagate: false
+  hdfs.client:
+    handlers: [main]
+    level: WARNING
+    propagate: false

{data_syncmaster-0.2.2 → data_syncmaster-0.2.4}/syncmaster/worker/spark.py

@@ -47,44 +47,41 @@ def get_worker_spark_session(
     return spark_builder.getOrCreate()
 
 
-def get_packages(connection_type: str) -> list[str]:  # noqa: WPS212
+def get_packages(connection_types: set[str]) -> list[str]:  # noqa: WPS212
     import pyspark
     from onetl.connection import MSSQL, Clickhouse, MySQL, Oracle, Postgres, SparkS3
     from onetl.file.format import XML, Excel
 
+    spark_version = pyspark.__version__
     # excel version is hardcoded due to https://github.com/nightscape/spark-excel/issues/902
     file_formats_spark_packages: list[str] = [
-        *XML.get_packages(spark_version=pyspark.__version__),
+        *XML.get_packages(spark_version=spark_version),
         *Excel.get_packages(spark_version="3.5.1"),
     ]
 
-    if connection_type == "postgres":
-        return Postgres.get_packages()
-    if connection_type == "oracle":
-        return Oracle.get_packages()
-    if connection_type == "clickhouse":
-        return [
-            "io.github.mtsongithub.doetl:spark-dialect-extension_2.12:0.0.2",
-            *Clickhouse.get_packages(),
-        ]
-    if connection_type == "mssql":
-        return MSSQL.get_packages()
-    if connection_type == "mysql":
-        return MySQL.get_packages()
-    if connection_type == "s3":
-        import pyspark
-
-        spark_version = pyspark.__version__
-        return SparkS3.get_packages(spark_version=spark_version) + file_formats_spark_packages
-
-    if connection_type in ("hdfs", "sftp", "ftp", "ftps", "samba", "webdav"):
-        return file_formats_spark_packages
-
-    # If the database type does not require downloading .jar packages
-    return []
-
-
-def get_excluded_packages(db_type: str) -> list[str]:
+    result = []
+    if connection_types & {"postgres", "all"}:
+        result.extend(Postgres.get_packages())
+    if connection_types & {"oracle", "all"}:
+        result.extend(Oracle.get_packages())
+    if connection_types & {"clickhouse", "all"}:
+        result.append("io.github.mtsongithub.doetl:spark-dialect-extension_2.12:0.0.2")
+        result.extend(Clickhouse.get_packages())
+    if connection_types & {"mssql", "all"}:
+        result.extend(MSSQL.get_packages())
+    if connection_types & {"mysql", "all"}:
+        result.extend(MySQL.get_packages())
+
+    if connection_types & {"s3", "all"}:
+        result.extend(SparkS3.get_packages(spark_version=spark_version))
+
+    if connection_types & {"s3", "hdfs", "sftp", "ftp", "ftps", "samba", "webdav", "all"}:
+        result.extend(file_formats_spark_packages)
+
+    return result
+
+
+def get_excluded_packages() -> list[str]:
     from onetl.connection import SparkS3
 
     return SparkS3.get_exclude_packages()
@@ -95,16 +92,11 @@ def get_spark_session_conf(
     target: ConnectionDTO,
     resources: dict,
 ) -> dict:
-    maven_packages: list[str] = []
-    excluded_packages: list[str] = []
-
-    for db_type in source, target:
-        maven_packages.extend(get_packages(connection_type=db_type.type))  # type: ignore
-        excluded_packages.extend(get_excluded_packages(db_type=db_type.type))  # type: ignore
+    maven_packages: list[str] = get_packages(connection_types={source.type, target.type})
+    excluded_packages: list[str] = get_excluded_packages()
 
     memory_mb = math.ceil(resources["ram_bytes_per_task"] / 1024 / 1024)
     config = {
-        "spark.jars.packages": ",".join(maven_packages),
         "spark.sql.pyspark.jvmStacktrace.enabled": "true",
         "spark.hadoop.mapreduce.fileoutputcommitter.marksuccessfuljobs": "false",
         "spark.executor.cores": resources["cpu_cores_per_task"],

data_syncmaster-0.2.4/syncmaster/worker/transfer.py

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from datetime import datetime, timezone
+
+from asgi_correlation_id import correlation_id
+from asgi_correlation_id.extensions.celery import load_correlation_ids
+from celery.signals import after_setup_task_logger
+from celery.utils.log import get_task_logger
+from jinja2 import Template
+from sqlalchemy import Engine, select
+from sqlalchemy.orm import Session, selectinload
+
+from syncmaster.db.models import AuthData, Run, Status, Transfer
+from syncmaster.db.repositories.utils import decrypt_auth_data
+from syncmaster.settings.log import setup_logging
+from syncmaster.worker.base import WorkerTask
+from syncmaster.worker.celery import app as celery
+from syncmaster.worker.controller import TransferController
+from syncmaster.worker.settings import WorkerAppSettings
+
+logger = get_task_logger(__name__)
+load_correlation_ids()
+
+
+@celery.task(name="run_transfer_task", bind=True, track_started=True)
+def run_transfer_task(self: WorkerTask, run_id: int) -> None:
+    run_transfer(
+        run_id=run_id,
+        engine=self.engine,
+        settings=self.settings,
+    )
+
+
+def run_transfer(run_id: int, engine: Engine, settings: WorkerAppSettings):
+    try:
+        with Session(engine, expire_on_commit=False) as session:
+            run = session.scalar(
+                select(Run)
+                .where(
+                    Run.id == run_id,
+                    Run.status == Status.CREATED,
+                )
+                .options(
+                    selectinload(Run.transfer),
+                    selectinload(Run.transfer).selectinload(Transfer.group),
+                    selectinload(Run.transfer).selectinload(Transfer.source_connection),
+                    selectinload(Run.transfer).selectinload(Transfer.target_connection),
+                ),
+            )
+            if run is None:
+                logger.info("Run %r not found or already executed by someone else", run_id)
+                return
+
+            source_connection = run.transfer.source_connection
+            source_auth_data_query = select(AuthData).where(AuthData.connection_id == run.transfer.source_connection.id)
+            source_auth_data_encrypted = session.scalar(source_auth_data_query)
+            source_auth_data = decrypt_auth_data(source_auth_data_encrypted.value, settings)
+
+            target_connection = run.transfer.target_connection
+            target_auth_data_query = select(AuthData).where(AuthData.connection_id == run.transfer.target_connection.id)
+            target_auth_data_encrypted = session.scalar(target_auth_data_query)
+            target_auth_data = decrypt_auth_data(target_auth_data_encrypted.value, settings)
+
+            logger.info("Starting run %r", run_id)
+            run.status = Status.STARTED
+            run.started_at = datetime.now(tz=timezone.utc)
+            run.log_url = Template(settings.worker.log_url_template).render(
+                run=run,
+                correlation_id=correlation_id.get(),
+            )
+            session.commit()
+            session.expunge_all()  # do not hold open DB connection while run is executed
+
+        controller = TransferController(
+            settings=settings,
+            run=run,
+            source_connection=source_connection,
+            target_connection=target_connection,
+            source_auth_data=source_auth_data,
+            target_auth_data=target_auth_data,
+        )
+        controller.perform_transfer()
+    except Exception as e:
+        status = Status.FAILED
+        logger.error("Run %r failed", run_id)
+        exception = e
+    else:
+        status = Status.FINISHED
+        logger.info("Run %r was successful", run_id)
+        exception = None
+
+    with Session(engine) as session:
+        run = session.get(Run, run_id)
+        if run:
+            logger.info("Updating run %r status in DB", run_id)
+            run.status = status
+            run.ended_at = datetime.now(tz=timezone.utc)
+            session.commit()
+
+    if exception is not None:
+        raise exception
+
+
+@after_setup_task_logger.connect
+def setup_loggers(*args, **kwargs):
+    setup_logging(WorkerAppSettings().logging.get_log_config_path())

data_syncmaster-0.2.2/syncmaster/worker/transfer.py

@@ -1,87 +0,0 @@
-# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
-# SPDX-License-Identifier: Apache-2.0
-from datetime import datetime, timezone
-
-from asgi_correlation_id import correlation_id
-from asgi_correlation_id.extensions.celery import load_correlation_ids
-from celery import Celery
-from celery.signals import after_setup_task_logger
-from celery.utils.log import get_task_logger
-from jinja2 import Template
-from sqlalchemy import select
-from sqlalchemy.orm import Session, selectinload
-
-from syncmaster.db.models import AuthData, Run, Status, Transfer
-from syncmaster.db.repositories.utils import decrypt_auth_data
-from syncmaster.exceptions.run import RunNotFoundError
-from syncmaster.settings.log import setup_logging
-from syncmaster.worker.celery import app as celery
-from syncmaster.worker.controller import TransferController
-from syncmaster.worker.settings import WorkerAppSettings
-
-logger = get_task_logger(__name__)
-load_correlation_ids()
-
-
-@celery.task(name="run_transfer_task", bind=True, track_started=True)
-def run_transfer_task(self: Celery, run_id: int) -> None:
-    with Session(self.engine) as session:
-        run_transfer(
-            session=session,
-            run_id=run_id,
-            settings=self.settings,
-        )
-
-
-def run_transfer(session: Session, run_id: int, settings: WorkerAppSettings):
-    logger.info("Start transfer")
-    run = session.get(
-        Run,
-        run_id,
-        options=(
-            selectinload(Run.transfer),
-            selectinload(Run.transfer).selectinload(Transfer.group),
-            selectinload(Run.transfer).selectinload(Transfer.source_connection),
-            selectinload(Run.transfer).selectinload(Transfer.target_connection),
-        ),
-    )
-    if run is None:
-        raise RunNotFoundError
-
-    run.status = Status.STARTED
-    run.started_at = datetime.now(tz=timezone.utc)
-    run.log_url = Template(settings.worker.log_url_template).render(run=run, correlation_id=correlation_id.get())
-    session.add(run)
-    session.commit()
-
-    q_source_auth_data = select(AuthData).where(AuthData.connection_id == run.transfer.source_connection.id)
-    q_target_auth_data = select(AuthData).where(AuthData.connection_id == run.transfer.target_connection.id)
-
-    target_auth_data = decrypt_auth_data(session.scalars(q_target_auth_data).one().value, settings)
-    source_auth_data = decrypt_auth_data(session.scalars(q_source_auth_data).one().value, settings)
-
-    try:
-        controller = TransferController(
-            settings=settings,
-            run=run,
-            source_connection=run.transfer.source_connection,
-            target_connection=run.transfer.target_connection,
-            source_auth_data=source_auth_data,
-            target_auth_data=target_auth_data,
-        )
-        controller.perform_transfer()
-    except Exception:
-        run.status = Status.FAILED
-        logger.exception("Run %r was failed", run.id)
-    else:
-        run.status = Status.FINISHED
-        logger.warning("Run %r was successful", run.id)
-
-    run.ended_at = datetime.now(tz=timezone.utc)
-    session.add(run)
-    session.commit()
-
-
-@after_setup_task_logger.connect
-def setup_loggers(*args, **kwargs):
-    setup_logging(WorkerAppSettings().logging.get_log_config_path())