data-syncmaster 0.3.2__tar.gz → 0.3.3__tar.gz

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-syncmaster
-Version: 0.3.2
+Version: 0.3.3
 Summary: Syncmaster REST API + scheduler + Worker
 Author-email: MWS Data Bridge <onetools@mts.ru>
 License-Expression: Apache-2.0
@@ -26,45 +26,46 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE.txt
-Requires-Dist: pydantic~=2.12.5
-Requires-Dist: pydantic-settings<2.14,>=2.12
+Requires-Dist: pydantic~=2.13.4
+Requires-Dist: pydantic-settings~=2.14.1
 Requires-Dist: pydantic-settings-logging~=0.1.1
-Requires-Dist: sqlalchemy~=2.0.44
+Requires-Dist: sqlalchemy~=2.0.49
 Requires-Dist: sqlalchemy-utils~=0.42.0
 Requires-Dist: pyyaml~=6.0.3
 Requires-Dist: coloredlogs~=15.0.1
-Requires-Dist: python-json-logger~=4.0.0
+Requires-Dist: python-json-logger~=4.1.0
 Requires-Dist: asgi-correlation-id~=4.3.4
 Requires-Dist: uuid6~=2025.0.1
 Requires-Dist: celery~=5.6.0
 Requires-Dist: psycopg2-binary~=2.9.11
 Provides-Extra: server
-Requires-Dist: fastapi~=0.129; extra == "server"
+Requires-Dist: fastapi<0.137.0,>=0.135.3; extra == "server"
+Requires-Dist: starlette<1.0; extra == "server"
 Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
-Requires-Dist: python-multipart~=0.0.21; extra == "server"
-Requires-Dist: uvicorn~=0.41.0; extra == "server"
-Requires-Dist: alembic~=1.18.1; extra == "server"
-Requires-Dist: pyjwt~=2.11.0; extra == "server"
-Requires-Dist: asyncpg~=0.31; extra == "server"
-Requires-Dist: python-keycloak<7.2.0,>=7.0.1; extra == "server"
+Requires-Dist: python-multipart~=0.0.26; extra == "server"
+Requires-Dist: uvicorn<0.48,>=0.44; extra == "server"
+Requires-Dist: alembic~=1.18.4; extra == "server"
+Requires-Dist: pyjwt~=2.12.1; extra == "server"
+Requires-Dist: asyncpg~=0.31.0; extra == "server"
+Requires-Dist: python-keycloak~=7.1.1; extra == "server"
 Requires-Dist: itsdangerous~=2.2.0; extra == "server"
 Provides-Extra: scheduler
 Requires-Dist: asyncpg~=0.31; extra == "scheduler"
 Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
 Provides-Extra: worker
-Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
+Requires-Dist: onetl[files]~=0.16.0; extra == "worker"
 Requires-Dist: pyspark<4.0.0; extra == "worker"
 Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
 Requires-Dist: jinja2~=3.1.6; extra == "worker"
 Provides-Extra: kerberos
-Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
+Requires-Dist: onetl[kerberos]~=0.16.0; extra == "kerberos"
 Dynamic: license-file
 
 .. _readme:
 
 |Logo|
 
-.. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/b6622b1b139137070f2287e288a4105c1110c1e4/docs/_static/logo_wide.svg
+.. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/0eb2ba2be6656f81ea9706523c3b7942df212640/docs/_static/logo_wide.svg
     :alt: Data.SyncMaster logo
     :target: https://github.com/MTSWebServices/data-syncmaster
 

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/README.rst

@@ -2,7 +2,7 @@
 
 |Logo|
 
-.. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/b6622b1b139137070f2287e288a4105c1110c1e4/docs/_static/logo_wide.svg
+.. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/0eb2ba2be6656f81ea9706523c3b7942df212640/docs/_static/logo_wide.svg
     :alt: Data.SyncMaster logo
     :target: https://github.com/MTSWebServices/data-syncmaster
 

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/data_syncmaster.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-syncmaster
-Version: 0.3.2
+Version: 0.3.3
 Summary: Syncmaster REST API + scheduler + Worker
 Author-email: MWS Data Bridge <onetools@mts.ru>
 License-Expression: Apache-2.0
@@ -26,45 +26,46 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE.txt
-Requires-Dist: pydantic~=2.12.5
-Requires-Dist: pydantic-settings<2.14,>=2.12
+Requires-Dist: pydantic~=2.13.4
+Requires-Dist: pydantic-settings~=2.14.1
 Requires-Dist: pydantic-settings-logging~=0.1.1
-Requires-Dist: sqlalchemy~=2.0.44
+Requires-Dist: sqlalchemy~=2.0.49
 Requires-Dist: sqlalchemy-utils~=0.42.0
 Requires-Dist: pyyaml~=6.0.3
 Requires-Dist: coloredlogs~=15.0.1
-Requires-Dist: python-json-logger~=4.0.0
+Requires-Dist: python-json-logger~=4.1.0
 Requires-Dist: asgi-correlation-id~=4.3.4
 Requires-Dist: uuid6~=2025.0.1
 Requires-Dist: celery~=5.6.0
 Requires-Dist: psycopg2-binary~=2.9.11
 Provides-Extra: server
-Requires-Dist: fastapi~=0.129; extra == "server"
+Requires-Dist: fastapi<0.137.0,>=0.135.3; extra == "server"
+Requires-Dist: starlette<1.0; extra == "server"
 Requires-Dist: starlette-exporter~=0.23.0; extra == "server"
-Requires-Dist: python-multipart~=0.0.21; extra == "server"
-Requires-Dist: uvicorn~=0.41.0; extra == "server"
-Requires-Dist: alembic~=1.18.1; extra == "server"
-Requires-Dist: pyjwt~=2.11.0; extra == "server"
-Requires-Dist: asyncpg~=0.31; extra == "server"
-Requires-Dist: python-keycloak<7.2.0,>=7.0.1; extra == "server"
+Requires-Dist: python-multipart~=0.0.26; extra == "server"
+Requires-Dist: uvicorn<0.48,>=0.44; extra == "server"
+Requires-Dist: alembic~=1.18.4; extra == "server"
+Requires-Dist: pyjwt~=2.12.1; extra == "server"
+Requires-Dist: asyncpg~=0.31.0; extra == "server"
+Requires-Dist: python-keycloak~=7.1.1; extra == "server"
 Requires-Dist: itsdangerous~=2.2.0; extra == "server"
 Provides-Extra: scheduler
 Requires-Dist: asyncpg~=0.31; extra == "scheduler"
 Requires-Dist: apscheduler~=3.11.1; extra == "scheduler"
 Provides-Extra: worker
-Requires-Dist: onetl[files]~=0.15.0; extra == "worker"
+Requires-Dist: onetl[files]~=0.16.0; extra == "worker"
 Requires-Dist: pyspark<4.0.0; extra == "worker"
 Requires-Dist: horizon-hwm-store~=1.1.4; extra == "worker"
 Requires-Dist: jinja2~=3.1.6; extra == "worker"
 Provides-Extra: kerberos
-Requires-Dist: onetl[kerberos]~=0.15.0; extra == "kerberos"
+Requires-Dist: onetl[kerberos]~=0.16.0; extra == "kerberos"
 Dynamic: license-file
 
 .. _readme:
 
 |Logo|
 
-.. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/b6622b1b139137070f2287e288a4105c1110c1e4/docs/_static/logo_wide.svg
+.. |Logo| image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/0eb2ba2be6656f81ea9706523c3b7942df212640/docs/_static/logo_wide.svg
     :alt: Data.SyncMaster logo
     :target: https://github.com/MTSWebServices/data-syncmaster
 

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/data_syncmaster.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ syncmaster/db/migrations/versions/2023-11-23_0009_create_celery_tables.py
 syncmaster/db/migrations/versions/2024-10-07_0010_add_pg_trgm_extension.py
 syncmaster/db/migrations/versions/2024-11-01_0011_create_apscheduler_table.py
 syncmaster/db/migrations/versions/2025-08-10_0012_update_ts.py
+syncmaster/db/migrations/versions/2026-04-02_0013_add_celery_date_done_indexes.py
 syncmaster/db/migrations/versions/__init__.py
 syncmaster/db/mixins/__init__.py
 syncmaster/db/mixins/resource.py

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/data_syncmaster.egg-info/requires.txt

@@ -1,36 +1,37 @@
-pydantic~=2.12.5
-pydantic-settings<2.14,>=2.12
+pydantic~=2.13.4
+pydantic-settings~=2.14.1
 pydantic-settings-logging~=0.1.1
-sqlalchemy~=2.0.44
+sqlalchemy~=2.0.49
 sqlalchemy-utils~=0.42.0
 pyyaml~=6.0.3
 coloredlogs~=15.0.1
-python-json-logger~=4.0.0
+python-json-logger~=4.1.0
 asgi-correlation-id~=4.3.4
 uuid6~=2025.0.1
 celery~=5.6.0
 psycopg2-binary~=2.9.11
 
 [kerberos]
-onetl[kerberos]~=0.15.0
+onetl[kerberos]~=0.16.0
 
 [scheduler]
 asyncpg~=0.31
 apscheduler~=3.11.1
 
 [server]
-fastapi~=0.129
+fastapi<0.137.0,>=0.135.3
+starlette<1.0
 starlette-exporter~=0.23.0
-python-multipart~=0.0.21
-uvicorn~=0.41.0
-alembic~=1.18.1
-pyjwt~=2.11.0
-asyncpg~=0.31
-python-keycloak<7.2.0,>=7.0.1
+python-multipart~=0.0.26
+uvicorn<0.48,>=0.44
+alembic~=1.18.4
+pyjwt~=2.12.1
+asyncpg~=0.31.0
+python-keycloak~=7.1.1
 itsdangerous~=2.2.0
 
 [worker]
-onetl[files]~=0.15.0
+onetl[files]~=0.16.0
 pyspark<4.0.0
 horizon-hwm-store~=1.1.4
 jinja2~=3.1.6

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/pyproject.toml

@@ -33,14 +33,14 @@ classifiers = [
 keywords = ["Syncmaster", "REST", "API", "Worker", "Spark", "Transfer", "ETL"]
 requires-python = ">=3.11"
 dependencies = [
-    "pydantic~=2.12.5",
-    "pydantic-settings>=2.12,<2.14",
+    "pydantic~=2.13.4",
+    "pydantic-settings~=2.14.1",
     "pydantic-settings-logging~=0.1.1",
-    "sqlalchemy~=2.0.44",
+    "sqlalchemy~=2.0.49",
     "sqlalchemy-utils~=0.42.0",
     "pyyaml~=6.0.3",
     "coloredlogs~=15.0.1",
-    "python-json-logger~=4.0.0",
+    "python-json-logger~=4.1.0",
     "asgi-correlation-id~=4.3.4",
     "uuid6~=2025.0.1",
     "celery~=5.6.0",
@@ -65,14 +65,16 @@ exclude = ["docs", "tests"]
 
 [project.optional-dependencies]
 server = [
-    "fastapi~=0.129",
+    "fastapi>=0.135.3,<0.137.0",
+    # starlette 1.0.0 break cookies
+    "starlette<1.0",
     "starlette-exporter~=0.23.0",
-    "python-multipart~=0.0.21",
-    "uvicorn~=0.41.0",
-    "alembic~=1.18.1",
-    "pyjwt~=2.11.0",
-    "asyncpg~=0.31",
-    "python-keycloak>=7.0.1,<7.2.0",
+    "python-multipart~=0.0.26",
+    "uvicorn>=0.44,<0.48",
+    "alembic~=1.18.4",
+    "pyjwt~=2.12.1",
+    "asyncpg~=0.31.0",
+    "python-keycloak~=7.1.1",
     "itsdangerous~=2.2.0",
 ]
 scheduler = [
@@ -80,13 +82,13 @@ scheduler = [
     "apscheduler~=3.11.1",
 ]
 worker = [
-    "onetl[files]~=0.15.0",
+    "onetl[files]~=0.16.0",
     "pyspark<4.0.0",
     "horizon-hwm-store~=1.1.4",
     "jinja2~=3.1.6",
 ]
 kerberos = [
-    "onetl[kerberos]~=0.15.0",
+    "onetl[kerberos]~=0.16.0",
 ]
 
 [dependency-groups]
@@ -94,36 +96,36 @@ test = [
     "pytest~=8.4.2",
     "httpx~=0.28.1",
     "pytest-asyncio>=0.26.0,<1.0",
-    "pytest-randomly~=4.0.1",
+    "pytest-randomly>=4.0.1,<4.2.0",
     "pytest-rerunfailures~=16.1",
-    "pytest-deadfixtures>=3.0,<3.2",
+    "pytest-deadfixtures~=3.1.0",
     "pytest-mock~=3.15.1",
     "pytest-lazy-fixtures~=1.4.0",
-    "faker~=40.1",
-    "coverage~=7.13.0",
-    "gevent~=25.9.1",
-    "respx~=0.22.0",
+    "faker~=40.13",
+    "coverage~=7.14.0",
+    "gevent>=26.4,<26.6",
+    "respx~=0.23.1",
     "dirty-equals~=0.11.0",
 ]
 dev = [
-    "ruff~=0.15.0",
-    "mypy~=1.19.1",
-    "prek~=0.3.2",
-    "sqlalchemy[mypy]~=2.0.44",
+    "ruff~=0.15.12",
+    "mypy~=2.1.0",
+    "prek>=0.3.2,<0.5.0",
+    "sqlalchemy[mypy]~=2.0.49",
     "types-jwcrypto~=1.5.0",
 ]
 docs = [
-  "setuptools-git-versioning>=2.1,<3.1",
+  "setuptools-git-versioning>=3.0.1,<3.2.0",
   "autodoc-pydantic~=2.2.0",
   "numpydoc~=1.10.0",
-  "sphinx>=8.2.3,<9.1.0",
+  "sphinx~=9.0.4",
   "furo~=2025.12.19",
   "sphinx-copybutton~=0.5.2",
   "sphinxcontrib-towncrier~=0.5.0a0",
   "towncrier~=25.8.0",
-  "sphinx-issues~=5.0.1",
-  "sphinx-design>=0.6.1,<0.8.0",
-  "sphinx-favicon>=1.0.1,<1.2.0",
+  "sphinx-issues~=6.0.0",
+  "sphinx-design~=0.7.0",
+  "sphinx-favicon~=1.1.0",
   "sphinx-argparse~=0.5.2",
   # https://github.com/mgeier/sphinx-last-updated-by-git/pull/77
   "sphinx-last-updated-by-git~=0.3.8",

data_syncmaster-0.3.3/syncmaster/VERSION

@@ -0,0 +1 @@
+0.3.3

data_syncmaster-0.3.3/syncmaster/db/migrations/versions/2026-04-02_0013_add_celery_date_done_indexes.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2025-present MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+"""Add celery_*_date_done indexex
+
+Revision ID: 0013
+Revises: 0012
+Create Date: 2026-04-02 11:06:44.293882
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0013"
+down_revision = "0012"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_index(
+        op.f("ix_celery_taskmeta_date_done"),
+        "celery_taskmeta",
+        ["date_done"],
+        unique=False,
+        if_not_exists=True,
+    )
+    op.create_index(
+        op.f("ix_celery_tasksetmeta_date_done"),
+        "celery_tasksetmeta",
+        ["date_done"],
+        unique=False,
+        if_not_exists=True,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(op.f("ix_celery_tasksetmeta_date_done"), table_name="celery_tasksetmeta", if_exists=True)
+    op.drop_index(op.f("ix_celery_taskmeta_date_done"), table_name="celery_taskmeta", if_exists=True)

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/syncmaster/worker/handlers/db/clickhouse.py

@@ -3,16 +3,36 @@
 
 from __future__ import annotations
 
+from datetime import UTC, datetime
+from decimal import Decimal
 from typing import TYPE_CHECKING, ClassVar
 
 from onetl.connection import Clickhouse
 from onetl.db import DBWriter
 from onetl.hooks import slot, support_hooks
+from pyspark.sql import functions as F  # noqa: N812
+from pyspark.sql.types import (
+    ArrayType,
+    BooleanType,
+    ByteType,
+    DataType,
+    DateType,
+    DecimalType,
+    DoubleType,
+    FloatType,
+    IntegerType,
+    LongType,
+    MapType,
+    ShortType,
+    StringType,
+    TimestampNTZType,
+    TimestampType,
+)
 
 from syncmaster.worker.handlers.db.base import DBHandler
 
 if TYPE_CHECKING:
-    from pyspark.sql import SparkSession
+    from pyspark.sql import Column, SparkSession
     from pyspark.sql.dataframe import DataFrame
 
     from syncmaster.dto.connections import ClickhouseConnectionDTO
@@ -38,6 +58,7 @@ class ClickhouseHandler(DBHandler):
             host=self.connection_dto.host,
             port=self.connection_dto.port,
             user=self.connection_dto.user,
+            database=self.connection_dto.database_name,
             password=self.connection_dto.password,
             extra=self.connection_dto.additional_params,
             spark=spark,
@@ -54,6 +75,7 @@ class ClickhouseHandler(DBHandler):
             (col for col in normalized_df.columns if col.lower().endswith("id")),
             normalized_df.columns[0],  # if there is no column with "id", take the first column
         )
+        normalized_df = self._normalize_column_to_non_nullable(normalized_df, sort_column)
         self.transfer_dto.options["createTableOptions"] = (
             f"ENGINE = MergeTree() ORDER BY {self._quote_field(sort_column)}"
         )
@@ -73,6 +95,36 @@ class ClickhouseHandler(DBHandler):
             df = df.withColumnRenamed(column_name, column_name.lower())
         return df
 
+    def _normalize_column_to_non_nullable(self, df: DataFrame, column: str) -> DataFrame:
+        """Rewrite ``column`` with ``coalesce`` so Spark marks it non-nullable.
+
+        TODO: remove this workaround if Spark infers ``nullable=false`` without a dummy literal:
+        https://issues.apache.org/jira/browse/SPARK-54302
+        """
+        field = df.schema[column]
+        if not field.nullable:
+            return df
+
+        field_type: DataType = field.dataType
+        sentinel: Column
+        if isinstance(field_type, (ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType, BooleanType)):
+            sentinel = F.lit(0).cast(field_type)
+        elif isinstance(field_type, DecimalType):
+            sentinel = F.lit(Decimal(0)).cast(field_type)
+        elif isinstance(field_type, StringType):
+            sentinel = F.lit("")
+        elif isinstance(field_type, (DateType, TimestampType, TimestampNTZType)):
+            sentinel = F.lit(datetime(1970, 1, 1, 0, 0, 0, tzinfo=UTC)).cast(field_type)
+        elif isinstance(field_type, ArrayType):
+            sentinel = F.array().cast(field_type)
+        elif isinstance(field_type, MapType):
+            sentinel = F.map_from_arrays(F.array(), F.array()).cast(field_type)
+        else:
+            msg = f"Unsupported Spark type for non-null: {field_type!r} (column {column!r})"
+            raise TypeError(msg)
+
+        return df.withColumn(column, F.coalesce(F.col(column), sentinel))
+
     def _make_rows_filter_expression(self, filters: list[dict]) -> str | None:
         expressions = []
         for filter_ in filters:

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/syncmaster/worker/handlers/db/mysql.py

@@ -35,6 +35,7 @@ class MySQLHandler(DBHandler):
             user=self.connection_dto.user,
             password=self.connection_dto.password,
             database=self.connection_dto.database_name,
+            extra=self.connection_dto.additional_params,
             spark=spark,
         ).check()
 

{data_syncmaster-0.3.2 → data_syncmaster-0.3.3}/syncmaster/worker/ivy2.py

@@ -31,7 +31,7 @@ def get_packages(connection_types: set[str]) -> list[str]:
     if connection_types & {"oracle", "all"}:
         result.extend(Oracle.get_packages())
     if connection_types & {"clickhouse", "all"}:
-        result.append("io.github.mtsongithub.doetl:spark-dialect-extension_2.12:0.0.2")
+        result.append("io.github.mtsongithub.doetl:spark-dialect-extension_2.12:0.0.4")
         result.extend(Clickhouse.get_packages())
     if connection_types & {"mssql", "all"}:
         result.extend(MSSQL.get_packages())

data_syncmaster-0.3.2/syncmaster/VERSION

@@ -1 +0,0 @@
-0.3.2