data-syncmaster 0.2.3__tar.gz → 0.2.4__tar.gz

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: data-syncmaster
-Version: 0.2.3
+Version: 0.2.4
 Summary: Syncmaster REST API + Worker
 License: Apache-2.0
 Keywords: Syncmaster,REST,API,Worker,Spark,Transfer,ETL
@@ -35,11 +35,12 @@ Requires-Dist: fastapi (>=0.115.12,<0.116.0) ; extra == "server"
 Requires-Dist: horizon-hwm-store (>=1.1.2,<2.0.0) ; extra == "worker"
 Requires-Dist: itsdangerous (>=2.2.0,<3.0.0) ; extra == "server"
 Requires-Dist: jinja2 (>=3.1.6,<4.0.0) ; extra == "server" or extra == "worker"
-Requires-Dist: onetl[all] (>=0.13.4,<0.14.0) ; extra == "worker"
+Requires-Dist: onetl[all] (>=0.13.5,<0.14.0) ; extra == "worker"
 Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0) ; extra == "server" or extra == "worker"
-Requires-Dist: pydantic (>=2.11.1,<3.0.0)
+Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: pyjwt (>=2.10.1,<3.0.0) ; extra == "server"
+Requires-Dist: pyspark (<4.0.0) ; extra == "worker"
 Requires-Dist: python-json-logger ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: python-keycloak (>=5.3.1,<6.0.0) ; extra == "server"
 Requires-Dist: python-multipart (>=0.0.20,<0.0.21) ; extra == "server"
@@ -47,8 +48,8 @@ Requires-Dist: pyyaml ; extra == "server" or extra == "worker" or extra == "sche
 Requires-Dist: sqlalchemy (>=2.0.40,<3.0.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: sqlalchemy-utils (>=0.41.2,<0.42.0) ; extra == "server" or extra == "worker" or extra == "scheduler"
 Requires-Dist: starlette-exporter (>=0.23.0,<0.24.0) ; extra == "server"
-Requires-Dist: uuid6 (>=2024.7.10,<2025.0.0) ; extra == "server" or extra == "worker"
-Requires-Dist: uvicorn (>=0.34.0,<0.35.0) ; extra == "server"
+Requires-Dist: uuid6 (>=2025.0.0,<2026.0.0) ; extra == "server" or extra == "worker"
+Requires-Dist: uvicorn (>=0.34,<0.36) ; extra == "server"
 Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
 Project-URL: Documentation, https://syncmaster.readthedocs.io
 Project-URL: Homepage, https://github.com/MobileTeleSystems/syncmaster

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "data-syncmaster"
-version = "0.2.3"
+version = "0.2.4"
 license = "Apache-2.0"
 description = "Syncmaster REST API + Worker"
 authors = ["DataOps.ETL <onetools@mts.ru>"]
@@ -44,22 +44,23 @@ exclude = [
 
 [tool.poetry.dependencies]
 python = "^3.11"
-pydantic = "^2.11.1"
+pydantic = "^2.11.7"
 pydantic-settings = { version = "^2.8.1", optional = true }
 sqlalchemy = { version = "^2.0.40", optional = true }
 sqlalchemy-utils = { version = "^0.41.2", optional = true }
 fastapi = { version = "^0.115.12", optional = true}
 asgi-correlation-id = {version = "^4.3.4", optional = true}
-uvicorn = { version = "^0.34.0", optional = true }
+uvicorn = { version = ">=0.34,<0.36", optional = true }
 alembic = { version = "^1.15.2", optional = true }
 pyjwt = { version = "^2.10.1", optional = true }
 jinja2 = { version = "^3.1.6", optional = true }
 python-multipart = { version = "^0.0.20", optional = true }
 celery = { version = "^5.5.0", optional = true }
-onetl = { version = "^0.13.4", extras = ["all"], optional = true }
+onetl = { version = "^0.13.5", extras = ["all"], optional = true }
+pyspark = { version = "<4.0.0", optional = true }
 pyyaml = { version = "*", optional = true }
 psycopg2-binary = { version = "^2.9.10", optional = true }
-uuid6 = {version = "^2024.7.10", optional = true}
+uuid6 = {version = "^2025.0.0", optional = true}
 coloredlogs = {version = "*", optional = true}
 python-json-logger = {version = "*", optional = true}
 asyncpg = { version = "^0.30.0", optional = true }
@@ -99,6 +100,7 @@ worker = [
     "sqlalchemy-utils",
     "celery",
     "onetl",
+    "pyspark",
     "asgi-correlation-id",
     "jinja2",
     "psycopg2-binary",
@@ -124,15 +126,15 @@ scheduler = [
 [tool.poetry.group.test.dependencies]
 pytest = "^8.3.5"
 httpx = "^0.28.1"
-pytest-asyncio = "^0.26.0"
+pytest-asyncio = ">=0.26.0,<1.0"
 pytest-randomly = "^3.15.0"
 pytest-rerunfailures = "^15.0"
 pytest-deadfixtures = "^2.2.1"
 pytest-mock = "^3.14.0"
 pytest-lazy-fixtures = "^1.1.1"
-faker = "^37.1.0"
-coverage = "^7.8.0"
-gevent = "^24.11.1"
+faker = "^37.4.0"
+coverage = "^7.9.1"
+gevent = ">=24.11.1,<26.0.0"
 responses = "^0.25.7"
 
 [tool.poetry.group.dev.dependencies]

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/syncmaster/__init__.py

@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 
-_raw_version = "0.2.3"
+_raw_version = "0.2.4"
 # version always contain only release number like 0.0.1
 __version__ = ".".join(_raw_version.split(".")[:3])  # noqa: WPS410

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/syncmaster/server/api/v1/runs.py

@@ -23,7 +23,6 @@ from syncmaster.schemas.v1.transfers.run import (
 from syncmaster.server.dependencies import Stub
 from syncmaster.server.services.get_user import get_user
 from syncmaster.server.services.unit_of_work import UnitOfWork
-from syncmaster.server.settings import ServerAppSettings as Settings
 
 router = APIRouter(tags=["Runs"], responses=get_error_responses())
 
@@ -82,7 +81,6 @@ async def read_run(
 @router.post("/runs")
 async def start_run(  # noqa: WPS217
     create_run_data: CreateRunSchema,
-    settings: Annotated[Settings, Depends(Stub(Settings))],
     celery: Annotated[Celery, Depends(Stub(Celery))],
     unit_of_work: UnitOfWork = Depends(UnitOfWork),
     current_user: User = Depends(get_user(is_active=True)),

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/syncmaster/server/middlewares/monitoring/metrics.py

@@ -37,7 +37,7 @@ def apply_monitoring_metrics_middleware(app: FastAPI, settings: MonitoringSettin
     app.add_middleware(
         PrometheusMiddleware,
         app_name=slugify(app.title),
-        skip_paths=skip_paths,
+        skip_paths=sorted(skip_paths),
         **settings.model_dump(exclude={"enabled", "skip_paths"}),
     )
     app.include_router(router)

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/syncmaster/server/middlewares/session.py

@@ -10,8 +10,8 @@ from syncmaster.server.settings.server.session import SessionSettings
 def apply_session_middleware(app: FastAPI, settings: SessionSettings) -> FastAPI:
     """Add SessionMiddleware middleware to the application."""
 
-    app.add_middleware(
-        SessionMiddleware,
-        **settings.model_dump(),
-    )
+    settings_dict = settings.model_dump(exclude={"secret_key"})
+    settings_dict["secret_key"] = settings.secret_key.get_secret_value()
+
+    app.add_middleware(SessionMiddleware, **settings_dict)
     return app

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/syncmaster/server/settings/auth/jwt.py

@@ -23,7 +23,11 @@ class JWTSettings(BaseModel):
             """
             Secret key for signing JWT tokens.
 
-            Can be any string. It is recommended to generate random value for every application instance.
+            Can be any string. It is recommended to generate random value for every application instance, e.g.:
+
+            .. code:: shell
+
+                pwgen 32 1
             """,
         ),
     )

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/syncmaster/server/settings/server/session.py

@@ -2,7 +2,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-from pydantic import BaseModel, ConfigDict, Field
+import textwrap
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
 
 
 class SessionSettings(BaseModel):
@@ -38,8 +40,23 @@ class SessionSettings(BaseModel):
 
     """
 
-    secret_key: str = Field(description="A random string for signing cookies.")
-    session_cookie: str | None = Field(default="session", description="Name of the session cookie.")
+    secret_key: SecretStr = Field(
+        description=textwrap.dedent(
+            """
+            Secret key for encrypting cookies.
+
+            Can be any string. It is recommended to generate random value for every application instance, e.g.:
+
+            .. code:: shell
+
+                pwgen 32 1
+            """,
+        ),
+    )
+    session_cookie: str | None = Field(
+        default="session",
+        description="Name of the session cookie. Change this if there are multiple application under the same domain.",
+    )
     max_age: int | None = Field(
         default=14 * 24 * 60 * 60,
         description="Session expiry time in seconds. Defaults to 2 weeks.",

{data_syncmaster-0.2.3 → data_syncmaster-0.2.4}/syncmaster/worker/spark.py

@@ -47,44 +47,41 @@ def get_worker_spark_session(
     return spark_builder.getOrCreate()
 
 
-def get_packages(connection_type: str) -> list[str]:  # noqa: WPS212
+def get_packages(connection_types: set[str]) -> list[str]:  # noqa: WPS212
     import pyspark
     from onetl.connection import MSSQL, Clickhouse, MySQL, Oracle, Postgres, SparkS3
     from onetl.file.format import XML, Excel
 
+    spark_version = pyspark.__version__
     # excel version is hardcoded due to https://github.com/nightscape/spark-excel/issues/902
     file_formats_spark_packages: list[str] = [
-        *XML.get_packages(spark_version=pyspark.__version__),
+        *XML.get_packages(spark_version=spark_version),
         *Excel.get_packages(spark_version="3.5.1"),
     ]
 
-    if connection_type == "postgres":
-        return Postgres.get_packages()
-    if connection_type == "oracle":
-        return Oracle.get_packages()
-    if connection_type == "clickhouse":
-        return [
-            "io.github.mtsongithub.doetl:spark-dialect-extension_2.12:0.0.2",
-            *Clickhouse.get_packages(),
-        ]
-    if connection_type == "mssql":
-        return MSSQL.get_packages()
-    if connection_type == "mysql":
-        return MySQL.get_packages()
-    if connection_type == "s3":
-        import pyspark
-
-        spark_version = pyspark.__version__
-        return SparkS3.get_packages(spark_version=spark_version) + file_formats_spark_packages
-
-    if connection_type in ("hdfs", "sftp", "ftp", "ftps", "samba", "webdav"):
-        return file_formats_spark_packages
-
-    # If the database type does not require downloading .jar packages
-    return []
-
-
-def get_excluded_packages(db_type: str) -> list[str]:
+    result = []
+    if connection_types & {"postgres", "all"}:
+        result.extend(Postgres.get_packages())
+    if connection_types & {"oracle", "all"}:
+        result.extend(Oracle.get_packages())
+    if connection_types & {"clickhouse", "all"}:
+        result.append("io.github.mtsongithub.doetl:spark-dialect-extension_2.12:0.0.2")
+        result.extend(Clickhouse.get_packages())
+    if connection_types & {"mssql", "all"}:
+        result.extend(MSSQL.get_packages())
+    if connection_types & {"mysql", "all"}:
+        result.extend(MySQL.get_packages())
+
+    if connection_types & {"s3", "all"}:
+        result.extend(SparkS3.get_packages(spark_version=spark_version))
+
+    if connection_types & {"s3", "hdfs", "sftp", "ftp", "ftps", "samba", "webdav", "all"}:
+        result.extend(file_formats_spark_packages)
+
+    return result
+
+
+def get_excluded_packages() -> list[str]:
     from onetl.connection import SparkS3
 
     return SparkS3.get_exclude_packages()
@@ -95,16 +92,11 @@ def get_spark_session_conf(
     target: ConnectionDTO,
     resources: dict,
 ) -> dict:
-    maven_packages: list[str] = []
-    excluded_packages: list[str] = []
-
-    for db_type in source, target:
-        maven_packages.extend(get_packages(connection_type=db_type.type))  # type: ignore
-        excluded_packages.extend(get_excluded_packages(db_type=db_type.type))  # type: ignore
+    maven_packages: list[str] = get_packages(connection_types={source.type, target.type})
+    excluded_packages: list[str] = get_excluded_packages()
 
     memory_mb = math.ceil(resources["ram_bytes_per_task"] / 1024 / 1024)
     config = {
-        "spark.jars.packages": ",".join(maven_packages),
         "spark.sql.pyspark.jvmStacktrace.enabled": "true",
         "spark.hadoop.mapreduce.fileoutputcommitter.marksuccessfuljobs": "false",
         "spark.executor.cores": resources["cpu_cores_per_task"],