data-syncmaster 0.1.3__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/PKG-INFO +8 -8
  2. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/pyproject.toml +48 -40
  3. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/__init__.py +1 -1
  4. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/__init__.py +4 -0
  5. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/connections.py +39 -58
  6. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/router.py +1 -1
  7. data_syncmaster-0.1.3/syncmaster/backend/api/v1/transfers/router.py → data_syncmaster-0.1.5/syncmaster/backend/api/v1/transfers.py +23 -41
  8. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/connection.py +4 -4
  9. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/credentials_repository.py +15 -20
  10. data_syncmaster-0.1.5/syncmaster/db/repositories/utils.py +37 -0
  11. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/dto/connections.py +7 -6
  12. data_syncmaster-0.1.5/syncmaster/dto/transfers.py +66 -0
  13. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/file/base.py +20 -9
  14. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/controller.py +22 -28
  15. data_syncmaster-0.1.5/syncmaster/worker/handlers/base.py +33 -0
  16. data_syncmaster-0.1.5/syncmaster/worker/handlers/db/base.py +38 -0
  17. data_syncmaster-0.1.5/syncmaster/worker/handlers/db/hive.py +37 -0
  18. {data_syncmaster-0.1.3/syncmaster/worker/handlers → data_syncmaster-0.1.5/syncmaster/worker/handlers/db}/oracle.py +14 -23
  19. {data_syncmaster-0.1.3/syncmaster/worker/handlers → data_syncmaster-0.1.5/syncmaster/worker/handlers/db}/postgres.py +14 -23
  20. data_syncmaster-0.1.5/syncmaster/worker/handlers/file/base.py +45 -0
  21. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/handlers/file/hdfs.py +12 -2
  22. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/handlers/file/s3.py +13 -2
  23. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/spark.py +18 -7
  24. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/transfer.py +11 -19
  25. data_syncmaster-0.1.3/syncmaster/backend/api/v1/transfers/utils.py +0 -17
  26. data_syncmaster-0.1.3/syncmaster/db/repositories/utils.py +0 -25
  27. data_syncmaster-0.1.3/syncmaster/dto/transfers.py +0 -46
  28. data_syncmaster-0.1.3/syncmaster/worker/handlers/base.py +0 -49
  29. data_syncmaster-0.1.3/syncmaster/worker/handlers/file/base.py +0 -56
  30. data_syncmaster-0.1.3/syncmaster/worker/handlers/hive.py +0 -41
  31. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/LICENSE.txt +0 -0
  32. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/README.rst +0 -0
  33. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/__main__.py +0 -0
  34. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/__init__.py +0 -0
  35. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/deps.py +0 -0
  36. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/monitoring.py +0 -0
  37. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/router.py +0 -0
  38. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/__init__.py +0 -0
  39. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/auth/__init__.py +0 -0
  40. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/auth/router.py +0 -0
  41. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/auth/utils.py +0 -0
  42. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/groups.py +0 -0
  43. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/queue.py +0 -0
  44. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/api/v1/users.py +0 -0
  45. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/export_openapi_schema.py +0 -0
  46. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/handler.py +0 -0
  47. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/services/__init__.py +0 -0
  48. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/services/auth.py +0 -0
  49. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/backend/services/unit_of_work.py +0 -0
  50. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/config.py +0 -0
  51. {data_syncmaster-0.1.3/syncmaster/backend/api/v1/transfers → data_syncmaster-0.1.5/syncmaster/db}/__init__.py +0 -0
  52. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/base.py +0 -0
  53. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/factory.py +0 -0
  54. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/migrations/README +0 -0
  55. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/migrations/__main__.py +0 -0
  56. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/migrations/alembic.ini +0 -0
  57. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/migrations/env.py +0 -0
  58. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/migrations/script.py.mako +0 -0
  59. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/migrations/versions/2023-11-23_478240cdad4b_init.py +0 -0
  60. {data_syncmaster-0.1.3/syncmaster/db → data_syncmaster-0.1.5/syncmaster/db/migrations/versions}/__init__.py +0 -0
  61. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/mixins.py +0 -0
  62. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/models.py +0 -0
  63. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/__init__.py +0 -0
  64. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/base.py +0 -0
  65. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/group.py +0 -0
  66. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/queue.py +0 -0
  67. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/repository_with_owner.py +0 -0
  68. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/run.py +0 -0
  69. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/transfer.py +0 -0
  70. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/repositories/user.py +0 -0
  71. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/db/utils.py +0 -0
  72. {data_syncmaster-0.1.3/syncmaster/db/migrations/versions → data_syncmaster-0.1.5/syncmaster/dto}/__init__.py +0 -0
  73. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/__init__.py +0 -0
  74. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/base.py +0 -0
  75. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/connection.py +0 -0
  76. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/credentials.py +0 -0
  77. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/group.py +0 -0
  78. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/queue.py +0 -0
  79. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/run.py +0 -0
  80. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/transfer.py +0 -0
  81. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/exceptions/user.py +0 -0
  82. {data_syncmaster-0.1.3/syncmaster/dto → data_syncmaster-0.1.5/syncmaster/schemas}/__init__.py +0 -0
  83. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/__init__.py +0 -0
  84. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/auth.py +0 -0
  85. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/connection_types.py +0 -0
  86. {data_syncmaster-0.1.3/syncmaster/schemas → data_syncmaster-0.1.5/syncmaster/schemas/v1/connections}/__init__.py +0 -0
  87. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/connections/connection.py +0 -0
  88. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/connections/hdfs.py +0 -0
  89. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/connections/hive.py +0 -0
  90. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/connections/oracle.py +0 -0
  91. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/connections/postgres.py +0 -0
  92. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/connections/s3.py +0 -0
  93. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/file_formats.py +0 -0
  94. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/groups.py +0 -0
  95. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/page.py +0 -0
  96. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/queue.py +0 -0
  97. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/status.py +0 -0
  98. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfer_types.py +0 -0
  99. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/__init__.py +0 -0
  100. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/db.py +0 -0
  101. {data_syncmaster-0.1.3/syncmaster/schemas/v1/connections → data_syncmaster-0.1.5/syncmaster/schemas/v1/transfers/file}/__init__.py +0 -0
  102. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/file/hdfs.py +0 -0
  103. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/file/s3.py +0 -0
  104. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/file_format.py +0 -0
  105. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/run.py +0 -0
  106. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/transfers/strategy.py +0 -0
  107. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/types.py +0 -0
  108. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/schemas/v1/users.py +0 -0
  109. {data_syncmaster-0.1.3/syncmaster/schemas/v1/transfers/file → data_syncmaster-0.1.5/syncmaster/worker}/__init__.py +0 -0
  110. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/base.py +0 -0
  111. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/config.py +0 -0
  112. {data_syncmaster-0.1.3/syncmaster/worker → data_syncmaster-0.1.5/syncmaster/worker/handlers}/__init__.py +0 -0
  113. {data_syncmaster-0.1.3/syncmaster/worker/handlers → data_syncmaster-0.1.5/syncmaster/worker/handlers/db}/__init__.py +0 -0
  114. {data_syncmaster-0.1.3 → data_syncmaster-0.1.5}/syncmaster/worker/handlers/file/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: data-syncmaster
3
- Version: 0.1.3
3
+ Version: 0.1.5
4
4
  Summary: Syncmaster REST API + Worker
5
5
  License: Apache-2.0
6
6
  Keywords: Syncmaster,REST,API,Worker,Replication
@@ -29,16 +29,16 @@ Provides-Extra: backend
29
29
  Provides-Extra: worker
30
30
  Requires-Dist: alembic (>=1.11.1,<2.0.0) ; extra == "backend"
31
31
  Requires-Dist: asyncpg (>=0.29.0,<0.30.0) ; extra == "backend"
32
- Requires-Dist: celery (>=5.3.3,<6.0.0)
32
+ Requires-Dist: celery (>=5.3.3,<6.0.0) ; extra == "backend" or extra == "worker"
33
33
  Requires-Dist: fastapi (>=0.110.0,<0.111.0) ; extra == "backend"
34
34
  Requires-Dist: onetl[spark] (>=0.10.2,<0.11.0) ; extra == "worker"
35
35
  Requires-Dist: psycopg2-binary (>=2.9.7,<3.0.0) ; extra == "worker"
36
- Requires-Dist: pydantic (>=2.6.4,<3.0.0)
37
- Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0)
38
- Requires-Dist: python-jose[cryptography] (>=3.3.0,<4.0.0)
39
- Requires-Dist: python-multipart (>=0.0.9,<0.0.10)
40
- Requires-Dist: sqlalchemy (>=2.0.18,<3.0.0)
41
- Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0)
36
+ Requires-Dist: pydantic (>=2.7.0,<3.0.0)
37
+ Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0) ; extra == "backend" or extra == "worker"
38
+ Requires-Dist: python-jose[cryptography] (>=3.3.0,<4.0.0) ; extra == "backend"
39
+ Requires-Dist: python-multipart (>=0.0.9,<0.0.10) ; extra == "backend"
40
+ Requires-Dist: sqlalchemy (>=2.0.18,<3.0.0) ; extra == "backend" or extra == "worker"
41
+ Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0) ; extra == "backend" or extra == "worker"
42
42
  Requires-Dist: uvicorn (>=0.29.0,<0.30.0) ; extra == "backend"
43
43
  Project-URL: CI/CD, https://github.com/MobileTeleSystems/syncmaster/actions
44
44
  Project-URL: Documentation, https://syncmaster.readthedocs.io
@@ -1,6 +1,10 @@
1
+ [build-system]
2
+ requires = ["poetry-core"]
3
+ build-backend = "poetry.core.masonry.api"
4
+
1
5
  [tool.poetry]
2
6
  name = "data-syncmaster"
3
- version = "0.1.3"
7
+ version = "0.1.5"
4
8
  license = "Apache-2.0"
5
9
  description = "Syncmaster REST API + Worker"
6
10
  authors = ["DataOps.ETL <onetools@mts.ru>"]
@@ -43,29 +47,40 @@ exclude = [
43
47
 
44
48
  [tool.poetry.dependencies]
45
49
  python = "^3.11"
46
- sqlalchemy = "^2.0.18"
47
- sqlalchemy-utils = "^0.41.1"
48
- pydantic = "^2.6.4"
49
- python-jose = {extras = ["cryptography"], version = "^3.3.0"}
50
- python-multipart = "^0.0.9"
51
- celery = "^5.3.3"
52
- onetl = {version = "^0.10.2", extras = ["spark"]}
53
- psycopg2-binary = {version = "^2.9.7", optional = true }
54
- fastapi = {version = "^0.110.0", optional = true}
55
- uvicorn = {version = "^0.29.0", optional = true }
56
- alembic = {version = "^1.11.1", optional = true }
57
- asyncpg = {version = "^0.29.0", optional = true }
58
- pydantic-settings = "^2.2.1"
50
+ pydantic = "^2.7.0"
51
+ pydantic-settings = { version = "^2.2.1", optional = true }
52
+ sqlalchemy = { version = "^2.0.18", optional = true }
53
+ sqlalchemy-utils = { version = "^0.41.1", optional = true }
54
+ fastapi = { version = "^0.110.0", optional = true}
55
+ uvicorn = { version = "^0.29.0", optional = true }
56
+ alembic = { version = "^1.11.1", optional = true }
57
+ asyncpg = { version = "^0.29.0", optional = true }
58
+ python-jose = { version = "^3.3.0", extras = ["cryptography"], optional = true }
59
+ python-multipart = { version = "^0.0.9", optional = true }
60
+ celery = { version = "^5.3.3", optional = true }
61
+ onetl = { version = "^0.10.2", extras = ["spark"], optional = true }
62
+ psycopg2-binary = { version = "^2.9.7", optional = true }
59
63
 
60
64
  [tool.poetry.extras]
61
65
  backend = [
62
- "alembic",
63
- "asyncpg",
66
+ "pydantic-settings",
67
+ "sqlalchemy",
68
+ "sqlalchemy-utils",
64
69
  "fastapi",
65
70
  "uvicorn",
71
+ "alembic",
72
+ "asyncpg",
73
+ "python-multipart",
74
+ "python-jose",
75
+ # migrations only
76
+ "celery",
66
77
  ]
67
78
 
68
79
  worker = [
80
+ "pydantic-settings",
81
+ "sqlalchemy",
82
+ "sqlalchemy-utils",
83
+ "celery",
69
84
  "onetl",
70
85
  "psycopg2-binary",
71
86
  ]
@@ -95,9 +110,23 @@ platformdirs = "4.2.0"
95
110
  sqlalchemy = {extras = ["mypy"], version = "^2.0.18"}
96
111
  types-python-jose = "^3.3.4.7"
97
112
 
98
- [build-system]
99
- requires = ["poetry-core"]
100
- build-backend = "poetry.core.masonry.api"
113
+ [tool.poetry.group.docs.dependencies]
114
+ autodoc-pydantic = {version = "^2.0.1", python = ">=3.8"}
115
+ numpydoc = {version = "^1.6.0", python = ">=3.8"}
116
+ sphinx = [
117
+ {version = "^7.1.2", python = ">=3.8"},
118
+ {version = "^7.2.6", python = ">=3.9"},
119
+ ]
120
+ furo = {version = "^2024.1.29", python = ">=3.8"}
121
+ sphinx-copybutton = {version = "^0.5.2", python = ">=3.8"}
122
+ sphinxcontrib-towncrier = {version = "^0.4.0a0", python = ">=3.8"}
123
+ towncrier = {version = "^23.11.0", python = ">=3.8"}
124
+ sphinx-issues = {version = ">=3.0.1,<5.0.0", python = ">=3.8"}
125
+ sphinx-design = {version = "^0.5.0", python = ">=3.8"}
126
+ sphinx-favicon = {version = "^1.0.1", python = ">=3.8"}
127
+ sphinx-argparse = {version = "^0.4.0", python = ">=3.8"}
128
+ # uncomment after https://github.com/zqmillet/sphinx-plantuml/pull/4
129
+ # sphinx-plantuml = {version = "^1.0.0", python = ">=3.8"}
101
130
 
102
131
  [tool.black]
103
132
  line-length = 120
@@ -197,27 +226,6 @@ exclude_lines = [
197
226
  "def downgrade\\(\\)",
198
227
  ]
199
228
 
200
-
201
-
202
-
203
- [tool.poetry.group.docs.dependencies]
204
- autodoc-pydantic = {version = "^2.0.1", python = ">=3.8"}
205
- numpydoc = {version = "^1.6.0", python = ">=3.8"}
206
- sphinx = [
207
- {version = "^7.1.2", python = ">=3.8"},
208
- {version = "^7.2.6", python = ">=3.9"},
209
- ]
210
- furo = {version = "^2024.1.29", python = ">=3.8"}
211
- sphinx-copybutton = {version = "^0.5.2", python = ">=3.8"}
212
- sphinxcontrib-towncrier = {version = "^0.4.0a0", python = ">=3.8"}
213
- towncrier = {version = "^23.11.0", python = ">=3.8"}
214
- sphinx-issues = {version = ">=3.0.1,<5.0.0", python = ">=3.8"}
215
- sphinx-design = {version = "^0.5.0", python = ">=3.8"}
216
- sphinx-favicon = {version = "^1.0.1", python = ">=3.8"}
217
- sphinx-argparse = {version = "^0.4.0", python = ">=3.8"}
218
- # uncomment after https://github.com/zqmillet/sphinx-plantuml/pull/4
219
- # sphinx-plantuml = {version = "^1.0.0", python = ">=3.8"}
220
-
221
229
  [tool.towncrier]
222
230
  name = "Syncmaster"
223
231
  package = "syncmaster"
@@ -1,6 +1,6 @@
1
1
  # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
 
4
- _raw_version = "0.1.3"
4
+ _raw_version = "0.1.5"
5
5
  # version always contain only release number like 0.0.1
6
6
  __version__ = ".".join(_raw_version.split(".")[:3]) # noqa: WPS410
@@ -54,3 +54,7 @@ def application_factory(settings: Settings) -> FastAPI:
54
54
  )
55
55
 
56
56
  return application
57
+
58
+
59
+ def get_application() -> FastAPI:
60
+ return application_factory(settings=Settings())
@@ -1,10 +1,8 @@
1
1
  # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
2
  # SPDX-License-Identifier: Apache-2.0
3
- import asyncio
4
3
  from typing import get_args
5
4
 
6
5
  from fastapi import APIRouter, Depends, Query, status
7
- from pydantic import SecretStr
8
6
 
9
7
  from syncmaster.backend.api.deps import UnitOfWorkMarker
10
8
  from syncmaster.backend.services import UnitOfWork, get_user
@@ -59,19 +57,17 @@ async def read_connections(
59
57
  items: list[ReadConnectionSchema] = []
60
58
 
61
59
  if pagination.items:
62
- creds = await asyncio.gather(
63
- *[unit_of_work.credentials.get_for_connection(connection_id=item.id) for item in pagination.items]
64
- )
60
+ credentials = await unit_of_work.credentials.read_bulk([item.id for item in pagination.items])
65
61
  items = [
66
62
  ReadConnectionSchema(
67
63
  id=item.id,
68
64
  group_id=item.group_id,
69
65
  name=item.name,
70
66
  description=item.description,
71
- auth_data=creds[n_item],
67
+ auth_data=credentials.get(item.id, None),
72
68
  data=item.data,
73
69
  )
74
- for n_item, item in enumerate(pagination.items)
70
+ for item in pagination.items
75
71
  ]
76
72
 
77
73
  return ConnectionPageSchema(
@@ -106,33 +102,27 @@ async def create_connection(
106
102
  if group_permission < Permission.WRITE:
107
103
  raise ActionNotAllowedError
108
104
 
109
- data = connection_data.data.dict()
110
- auth_data = connection_data.auth_data.dict()
111
-
112
- # Trick to serialize SecretStr to JSON
113
- for k, v in auth_data.items():
114
- if isinstance(v, SecretStr):
115
- auth_data[k] = v.get_secret_value()
116
105
  async with unit_of_work:
117
106
  connection = await unit_of_work.connection.create(
118
107
  name=connection_data.name,
119
108
  description=connection_data.description,
120
109
  group_id=connection_data.group_id,
121
- data=data,
110
+ data=connection_data.data.dict(),
122
111
  )
123
112
 
124
- await unit_of_work.credentials.add_to_connection(
113
+ await unit_of_work.credentials.create(
125
114
  connection_id=connection.id,
126
- data=auth_data,
115
+ data=connection_data.auth_data.dict(),
127
116
  )
128
117
 
118
+ credentials = await unit_of_work.credentials.read(connection.id)
129
119
  return ReadConnectionSchema(
130
120
  id=connection.id,
131
121
  group_id=connection.group_id,
132
122
  name=connection.name,
133
123
  description=connection.description,
134
124
  data=connection.data,
135
- auth_data=auth_data,
125
+ auth_data=credentials,
136
126
  )
137
127
 
138
128
 
@@ -155,12 +145,9 @@ async def read_connection(
155
145
  if resource_role == Permission.NONE:
156
146
  raise ConnectionNotFoundError
157
147
 
158
- connection = await unit_of_work.connection.read_by_id(connection_id=connection_id)
159
-
148
+ connection = await unit_of_work.connection.read_by_id(connection_id)
160
149
  try:
161
- credentials = await unit_of_work.credentials.get_for_connection(
162
- connection_id=connection.id,
163
- )
150
+ credentials = await unit_of_work.credentials.read(connection.id)
164
151
  except AuthDataNotFoundError:
165
152
  credentials = None
166
153
 
@@ -177,7 +164,7 @@ async def read_connection(
177
164
  @router.patch("/connections/{connection_id}")
178
165
  async def update_connection(
179
166
  connection_id: int,
180
- connection_data: UpdateConnectionSchema,
167
+ changes: UpdateConnectionSchema,
181
168
  current_user: User = Depends(get_user(is_active=True)),
182
169
  unit_of_work: UnitOfWork = Depends(UnitOfWorkMarker),
183
170
  ) -> ReadConnectionSchema:
@@ -195,25 +182,25 @@ async def update_connection(
195
182
  async with unit_of_work:
196
183
  connection = await unit_of_work.connection.update(
197
184
  connection_id=connection_id,
198
- name=connection_data.name,
199
- description=connection_data.description,
200
- connection_data=connection_data.data.dict(exclude={"auth_data"}) if connection_data.data else {},
185
+ name=changes.name,
186
+ description=changes.description,
187
+ data=changes.data.dict(exclude={"auth_data"}) if changes.data else {},
201
188
  )
202
189
 
203
- if connection_data.auth_data:
190
+ if changes.auth_data:
204
191
  await unit_of_work.credentials.update(
205
192
  connection_id=connection_id,
206
- credential_data=connection_data.auth_data.dict(),
193
+ data=changes.auth_data.dict(),
207
194
  )
208
195
 
209
- auth_data = await unit_of_work.credentials.get_for_connection(connection_id)
196
+ credentials = await unit_of_work.credentials.read(connection_id)
210
197
  return ReadConnectionSchema(
211
198
  id=connection.id,
212
199
  group_id=connection.group_id,
213
200
  name=connection.name,
214
201
  description=connection.description,
215
202
  data=connection.data,
216
- auth_data=auth_data,
203
+ auth_data=credentials,
217
204
  )
218
205
 
219
206
 
@@ -227,28 +214,26 @@ async def delete_connection(
227
214
  user=current_user,
228
215
  resource_id=connection_id,
229
216
  )
230
-
231
217
  if resource_role == Permission.NONE:
232
218
  raise ConnectionNotFoundError
233
219
 
234
220
  if resource_role < Permission.DELETE:
235
221
  raise ActionNotAllowedError
236
222
 
237
- connection = await unit_of_work.connection.read_by_id(connection_id=connection_id)
223
+ connection = await unit_of_work.connection.read_by_id(connection_id)
224
+ transfers = await unit_of_work.transfer.list_by_connection_id(connection.id)
225
+ if transfers:
226
+ raise ConnectionDeleteError(
227
+ f"The connection has an associated transfers. Number of the connected transfers: {len(transfers)}",
228
+ )
238
229
 
239
- transfers = await unit_of_work.transfer.list_by_connection_id(conn_id=connection.id)
240
230
  async with unit_of_work:
241
- if not transfers:
242
- await unit_of_work.connection.delete(connection_id=connection_id)
231
+ await unit_of_work.connection.delete(connection_id)
243
232
 
244
- return StatusResponseSchema(
245
- ok=True,
246
- status_code=status.HTTP_200_OK,
247
- message="Connection was deleted",
248
- )
249
-
250
- raise ConnectionDeleteError(
251
- f"The connection has an associated transfers. Number of the connected transfers: {len(transfers)}",
233
+ return StatusResponseSchema(
234
+ ok=True,
235
+ status_code=status.HTTP_200_OK,
236
+ message="Connection was deleted",
252
237
  )
253
238
 
254
239
 
@@ -259,24 +244,20 @@ async def copy_connection(
259
244
  current_user: User = Depends(get_user(is_active=True)),
260
245
  unit_of_work: UnitOfWork = Depends(UnitOfWorkMarker),
261
246
  ) -> StatusResponseSchema:
262
- target_source_rules = await asyncio.gather(
263
- unit_of_work.connection.get_resource_permission(
264
- user=current_user,
265
- resource_id=connection_id,
266
- ),
267
- unit_of_work.connection.get_group_permission(
268
- user=current_user,
269
- group_id=copy_connection_data.new_group_id,
270
- ),
247
+ resource_role = await unit_of_work.connection.get_resource_permission(
248
+ user=current_user,
249
+ resource_id=connection_id,
271
250
  )
272
- resource_role, target_group_role = target_source_rules
251
+ if resource_role == Permission.NONE:
252
+ raise ConnectionNotFoundError
273
253
 
274
254
  if copy_connection_data.remove_source and resource_role < Permission.DELETE:
275
255
  raise ActionNotAllowedError
276
256
 
277
- if resource_role == Permission.NONE:
278
- raise ConnectionNotFoundError
279
-
257
+ target_group_role = await unit_of_work.connection.get_group_permission(
258
+ user=current_user,
259
+ group_id=copy_connection_data.new_group_id,
260
+ )
280
261
  if target_group_role == Permission.NONE:
281
262
  raise GroupNotFoundError
282
263
 
@@ -291,7 +272,7 @@ async def copy_connection(
291
272
  )
292
273
 
293
274
  if copy_connection_data.remove_source:
294
- await unit_of_work.connection.delete(connection_id=connection_id)
275
+ await unit_of_work.connection.delete(connection_id)
295
276
 
296
277
  return StatusResponseSchema(
297
278
  ok=True,
@@ -6,7 +6,7 @@ from syncmaster.backend.api.v1.auth.router import router as auth_router
6
6
  from syncmaster.backend.api.v1.connections import router as connection_router
7
7
  from syncmaster.backend.api.v1.groups import router as group_router
8
8
  from syncmaster.backend.api.v1.queue import router as queue_router
9
- from syncmaster.backend.api.v1.transfers.router import router as transfer_router
9
+ from syncmaster.backend.api.v1.transfers import router as transfer_router
10
10
  from syncmaster.backend.api.v1.users import router as user_router
11
11
 
12
12
  router = APIRouter(prefix="/v1")
@@ -1,14 +1,10 @@
1
1
  # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
2
  # SPDX-License-Identifier: Apache-2.0
3
- import asyncio
4
3
 
5
4
  from fastapi import APIRouter, Depends, Query, status
6
5
  from kombu.exceptions import KombuError
7
6
 
8
7
  from syncmaster.backend.api.deps import UnitOfWorkMarker
9
- from syncmaster.backend.api.v1.transfers.utils import (
10
- process_file_transfer_directory_path,
11
- )
12
8
  from syncmaster.backend.services import UnitOfWork, get_user
13
9
  from syncmaster.db.models import Status, User
14
10
  from syncmaster.db.utils import Permission
@@ -79,16 +75,11 @@ async def create_transfer(
79
75
  user=current_user,
80
76
  group_id=transfer_data.group_id,
81
77
  )
82
-
83
78
  if group_permission < Permission.WRITE:
84
79
  raise ActionNotAllowedError
85
80
 
86
- target_connection = await unit_of_work.connection.read_by_id(
87
- connection_id=transfer_data.target_connection_id,
88
- )
89
- source_connection = await unit_of_work.connection.read_by_id(
90
- connection_id=transfer_data.source_connection_id,
91
- )
81
+ target_connection = await unit_of_work.connection.read_by_id(transfer_data.target_connection_id)
82
+ source_connection = await unit_of_work.connection.read_by_id(transfer_data.source_connection_id)
92
83
  queue = await unit_of_work.queue.read_by_id(transfer_data.queue_id)
93
84
 
94
85
  if (
@@ -115,8 +106,6 @@ async def create_transfer(
115
106
  if transfer_data.group_id != queue.group_id:
116
107
  raise DifferentTransferAndQueueGroupError
117
108
 
118
- transfer_data = process_file_transfer_directory_path(transfer_data) # type: ignore
119
-
120
109
  async with unit_of_work:
121
110
  transfer = await unit_of_work.transfer.create(
122
111
  group_id=transfer_data.group_id,
@@ -158,44 +147,39 @@ async def copy_transfer(
158
147
  current_user: User = Depends(get_user(is_active=True)),
159
148
  unit_of_work: UnitOfWork = Depends(UnitOfWorkMarker),
160
149
  ) -> StatusCopyTransferResponseSchema:
161
- # Check: user can copy transfer
162
- target_source_transfer_rules = await asyncio.gather(
163
- unit_of_work.transfer.get_resource_permission(
164
- user=current_user,
165
- resource_id=transfer_id,
166
- ),
167
- unit_of_work.transfer.get_group_permission(
168
- user=current_user,
169
- group_id=transfer_data.new_group_id,
170
- ),
150
+ resource_role = await unit_of_work.transfer.get_resource_permission(
151
+ user=current_user,
152
+ resource_id=transfer_id,
171
153
  )
172
- resource_role, target_group_role = target_source_transfer_rules
173
-
174
154
  if resource_role == Permission.NONE:
175
155
  raise TransferNotFoundError
176
156
 
177
- if target_group_role < Permission.WRITE:
178
- raise ActionNotAllowedError
179
-
180
157
  # Check: user can delete transfer
181
158
  if transfer_data.remove_source and resource_role < Permission.DELETE:
182
159
  raise ActionNotAllowedError
183
160
 
161
+ target_group_role = await unit_of_work.transfer.get_group_permission(
162
+ user=current_user,
163
+ group_id=transfer_data.new_group_id,
164
+ )
165
+ if target_group_role < Permission.WRITE:
166
+ raise ActionNotAllowedError
167
+
184
168
  transfer = await unit_of_work.transfer.read_by_id(transfer_id=transfer_id)
169
+
185
170
  # Check: user can copy connection
186
- target_source_connection_rules = await asyncio.gather(
187
- unit_of_work.connection.get_resource_permission(
188
- user=current_user,
189
- resource_id=transfer.source_connection_id,
190
- ),
191
- unit_of_work.connection.get_resource_permission(
192
- user=current_user,
193
- resource_id=transfer.target_connection_id,
194
- ),
171
+ source_connection_role = await unit_of_work.connection.get_resource_permission(
172
+ user=current_user,
173
+ resource_id=transfer.source_connection_id,
195
174
  )
196
- source_connection_role, target_connection_role = target_source_connection_rules
175
+ if source_connection_role == Permission.NONE:
176
+ raise ConnectionNotFoundError
197
177
 
198
- if source_connection_role == Permission.NONE or target_connection_role == Permission.NONE:
178
+ target_connection_role = await unit_of_work.connection.get_resource_permission(
179
+ user=current_user,
180
+ resource_id=transfer.target_connection_id,
181
+ )
182
+ if target_connection_role == Permission.NONE:
199
183
  raise ConnectionNotFoundError
200
184
 
201
185
  # Check: new queue exists
@@ -316,8 +300,6 @@ async def update_transfer(
316
300
  params_type=transfer_data.source_params.type,
317
301
  )
318
302
 
319
- transfer_data = process_file_transfer_directory_path(transfer_data) # type: ignore
320
-
321
303
  async with unit_of_work:
322
304
  transfer = await unit_of_work.transfer.update(
323
305
  transfer=transfer,
@@ -81,19 +81,19 @@ class ConnectionRepository(RepositoryWithOwner[Connection]):
81
81
  connection_id: int,
82
82
  name: str | None,
83
83
  description: str | None,
84
- connection_data: dict[str, Any],
84
+ data: dict[str, Any],
85
85
  ) -> Connection:
86
86
  try:
87
87
  connection = await self.read_by_id(connection_id=connection_id)
88
88
  for key in connection.data:
89
- if key not in connection_data or connection_data[key] is None:
90
- connection_data[key] = connection.data[key]
89
+ data[key] = data.get(key, None) or connection.data[key]
90
+
91
91
  return await self._update(
92
92
  Connection.id == connection_id,
93
93
  Connection.is_deleted.is_(False),
94
94
  name=name or connection.name,
95
95
  description=description or connection.description,
96
- data=connection_data,
96
+ data=data,
97
97
  )
98
98
  except IntegrityError as e:
99
99
  self._raise_error(e)
@@ -4,7 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  from typing import NoReturn
6
6
 
7
- from sqlalchemy import ScalarResult, delete, insert, select
7
+ from sqlalchemy import ScalarResult, insert, select
8
8
  from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound
9
9
  from sqlalchemy.ext.asyncio import AsyncSession
10
10
 
@@ -26,7 +26,7 @@ class CredentialsRepository(Repository[AuthData]):
26
26
  super().__init__(model=model, session=session)
27
27
  self._settings = settings
28
28
 
29
- async def get_for_connection(
29
+ async def read(
30
30
  self,
31
31
  connection_id: int,
32
32
  ) -> dict:
@@ -37,7 +37,15 @@ class CredentialsRepository(Repository[AuthData]):
37
37
  except NoResultFound as e:
38
38
  raise AuthDataNotFoundError(f"Connection id = {connection_id}") from e
39
39
 
40
- async def add_to_connection(self, connection_id: int, data: dict) -> AuthData:
40
+ async def read_bulk(
41
+ self,
42
+ connection_ids: list[int],
43
+ ) -> dict[int, dict]:
44
+ query = select(AuthData).where(AuthData.connection_id.in_(connection_ids))
45
+ result: ScalarResult[AuthData] = await self._session.scalars(query)
46
+ return {item.connection_id: decrypt_auth_data(item.value, settings=self._settings) for item in result}
47
+
48
+ async def create(self, connection_id: int, data: dict) -> AuthData:
41
49
  query = (
42
50
  insert(AuthData)
43
51
  .values(
@@ -54,31 +62,18 @@ class CredentialsRepository(Repository[AuthData]):
54
62
  await self._session.flush()
55
63
  return result.one()
56
64
 
57
- async def delete_from_connection(self, connection_id: int) -> AuthData:
58
- query = delete(AuthData).where(AuthData.connection_id == connection_id).returning(AuthData)
59
-
60
- try:
61
- result: ScalarResult[AuthData] = await self._session.scalars(query)
62
- except IntegrityError as e:
63
- self._raise_error(e)
64
- else:
65
- await self._session.flush()
66
- return result.one()
67
-
68
65
  async def update(
69
66
  self,
70
67
  connection_id: int,
71
- credential_data: dict,
68
+ data: dict,
72
69
  ) -> AuthData:
73
- creds = await self.get_for_connection(connection_id)
70
+ creds = await self.read(connection_id)
74
71
  try:
75
72
  for key in creds:
76
- if key not in credential_data or credential_data[key] is None:
77
- credential_data[key] = creds[key]
78
-
73
+ data[key] = data.get(key, None) or creds[key]
79
74
  return await self._update(
80
75
  AuthData.connection_id == connection_id,
81
- value=encrypt_auth_data(value=credential_data, settings=self._settings),
76
+ value=encrypt_auth_data(value=data, settings=self._settings),
82
77
  )
83
78
  except IntegrityError as e:
84
79
  self._raise_error(e)
@@ -0,0 +1,37 @@
1
+ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ import json
4
+
5
+ from cryptography.fernet import Fernet
6
+ from pydantic import SecretStr
7
+
8
+ from syncmaster.config import Settings
9
+
10
+
11
+ def decrypt_auth_data(
12
+ value: str,
13
+ settings: Settings,
14
+ ) -> dict:
15
+ decryptor = Fernet(settings.CRYPTO_KEY)
16
+ decrypted = decryptor.decrypt(value)
17
+ return json.loads(decrypted)
18
+
19
+
20
+ def _json_default(value):
21
+ if isinstance(value, SecretStr):
22
+ return value.get_secret_value()
23
+
24
+
25
+ def encrypt_auth_data(
26
+ value: dict,
27
+ settings: Settings,
28
+ ) -> str:
29
+ encryptor = Fernet(settings.CRYPTO_KEY)
30
+ serialized = json.dumps(
31
+ value,
32
+ ensure_ascii=False,
33
+ sort_keys=True,
34
+ default=_json_default,
35
+ )
36
+ encrypted = encryptor.encrypt(serialized.encode("utf-8"))
37
+ return encrypted.decode("utf-8")