opengris-scaler 1.12.7__cp311-cp311-manylinux_2_28_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of opengris-scaler might be problematic. Click here for more details.

Files changed (232) hide show
  1. opengris_scaler-1.12.7.dist-info/METADATA +729 -0
  2. opengris_scaler-1.12.7.dist-info/RECORD +232 -0
  3. opengris_scaler-1.12.7.dist-info/WHEEL +5 -0
  4. opengris_scaler-1.12.7.dist-info/entry_points.txt +9 -0
  5. opengris_scaler-1.12.7.dist-info/licenses/LICENSE +201 -0
  6. opengris_scaler-1.12.7.dist-info/licenses/LICENSE.spdx +7 -0
  7. opengris_scaler-1.12.7.dist-info/licenses/NOTICE +8 -0
  8. opengris_scaler.libs/libcapnp-1-b787335c.1.0.so +0 -0
  9. opengris_scaler.libs/libkj-1-094aa318.1.0.so +0 -0
  10. scaler/CMakeLists.txt +11 -0
  11. scaler/__init__.py +14 -0
  12. scaler/about.py +5 -0
  13. scaler/client/__init__.py +0 -0
  14. scaler/client/agent/__init__.py +0 -0
  15. scaler/client/agent/client_agent.py +210 -0
  16. scaler/client/agent/disconnect_manager.py +27 -0
  17. scaler/client/agent/future_manager.py +112 -0
  18. scaler/client/agent/heartbeat_manager.py +74 -0
  19. scaler/client/agent/mixins.py +89 -0
  20. scaler/client/agent/object_manager.py +98 -0
  21. scaler/client/agent/task_manager.py +64 -0
  22. scaler/client/client.py +635 -0
  23. scaler/client/future.py +252 -0
  24. scaler/client/object_buffer.py +129 -0
  25. scaler/client/object_reference.py +25 -0
  26. scaler/client/serializer/__init__.py +0 -0
  27. scaler/client/serializer/default.py +16 -0
  28. scaler/client/serializer/mixins.py +38 -0
  29. scaler/cluster/__init__.py +0 -0
  30. scaler/cluster/cluster.py +115 -0
  31. scaler/cluster/combo.py +148 -0
  32. scaler/cluster/object_storage_server.py +45 -0
  33. scaler/cluster/scheduler.py +83 -0
  34. scaler/config/__init__.py +0 -0
  35. scaler/config/defaults.py +87 -0
  36. scaler/config/loader.py +95 -0
  37. scaler/config/mixins.py +15 -0
  38. scaler/config/section/__init__.py +0 -0
  39. scaler/config/section/cluster.py +56 -0
  40. scaler/config/section/native_worker_adapter.py +44 -0
  41. scaler/config/section/object_storage_server.py +7 -0
  42. scaler/config/section/scheduler.py +53 -0
  43. scaler/config/section/symphony_worker_adapter.py +47 -0
  44. scaler/config/section/top.py +13 -0
  45. scaler/config/section/webui.py +16 -0
  46. scaler/config/types/__init__.py +0 -0
  47. scaler/config/types/object_storage_server.py +45 -0
  48. scaler/config/types/worker.py +57 -0
  49. scaler/config/types/zmq.py +79 -0
  50. scaler/entry_points/__init__.py +0 -0
  51. scaler/entry_points/cluster.py +133 -0
  52. scaler/entry_points/object_storage_server.py +41 -0
  53. scaler/entry_points/scheduler.py +135 -0
  54. scaler/entry_points/top.py +286 -0
  55. scaler/entry_points/webui.py +26 -0
  56. scaler/entry_points/worker_adapter_native.py +137 -0
  57. scaler/entry_points/worker_adapter_symphony.py +102 -0
  58. scaler/io/__init__.py +0 -0
  59. scaler/io/async_binder.py +85 -0
  60. scaler/io/async_connector.py +95 -0
  61. scaler/io/async_object_storage_connector.py +185 -0
  62. scaler/io/mixins.py +154 -0
  63. scaler/io/sync_connector.py +68 -0
  64. scaler/io/sync_object_storage_connector.py +185 -0
  65. scaler/io/sync_subscriber.py +83 -0
  66. scaler/io/utility.py +31 -0
  67. scaler/io/ymq/CMakeLists.txt +98 -0
  68. scaler/io/ymq/__init__.py +0 -0
  69. scaler/io/ymq/_ymq.pyi +96 -0
  70. scaler/io/ymq/_ymq.so +0 -0
  71. scaler/io/ymq/bytes.h +114 -0
  72. scaler/io/ymq/common.h +29 -0
  73. scaler/io/ymq/configuration.h +60 -0
  74. scaler/io/ymq/epoll_context.cpp +185 -0
  75. scaler/io/ymq/epoll_context.h +85 -0
  76. scaler/io/ymq/error.h +132 -0
  77. scaler/io/ymq/event_loop.h +55 -0
  78. scaler/io/ymq/event_loop_thread.cpp +64 -0
  79. scaler/io/ymq/event_loop_thread.h +46 -0
  80. scaler/io/ymq/event_manager.h +81 -0
  81. scaler/io/ymq/file_descriptor.h +203 -0
  82. scaler/io/ymq/interruptive_concurrent_queue.h +169 -0
  83. scaler/io/ymq/io_context.cpp +98 -0
  84. scaler/io/ymq/io_context.h +44 -0
  85. scaler/io/ymq/io_socket.cpp +299 -0
  86. scaler/io/ymq/io_socket.h +121 -0
  87. scaler/io/ymq/iocp_context.cpp +102 -0
  88. scaler/io/ymq/iocp_context.h +83 -0
  89. scaler/io/ymq/logging.h +163 -0
  90. scaler/io/ymq/message.h +15 -0
  91. scaler/io/ymq/message_connection.h +16 -0
  92. scaler/io/ymq/message_connection_tcp.cpp +672 -0
  93. scaler/io/ymq/message_connection_tcp.h +96 -0
  94. scaler/io/ymq/network_utils.h +179 -0
  95. scaler/io/ymq/pymod_ymq/bytes.h +113 -0
  96. scaler/io/ymq/pymod_ymq/exception.h +124 -0
  97. scaler/io/ymq/pymod_ymq/gil.h +15 -0
  98. scaler/io/ymq/pymod_ymq/io_context.h +166 -0
  99. scaler/io/ymq/pymod_ymq/io_socket.h +285 -0
  100. scaler/io/ymq/pymod_ymq/message.h +99 -0
  101. scaler/io/ymq/pymod_ymq/python.h +153 -0
  102. scaler/io/ymq/pymod_ymq/ymq.cpp +23 -0
  103. scaler/io/ymq/pymod_ymq/ymq.h +357 -0
  104. scaler/io/ymq/readme.md +114 -0
  105. scaler/io/ymq/simple_interface.cpp +80 -0
  106. scaler/io/ymq/simple_interface.h +24 -0
  107. scaler/io/ymq/tcp_client.cpp +367 -0
  108. scaler/io/ymq/tcp_client.h +75 -0
  109. scaler/io/ymq/tcp_operations.h +41 -0
  110. scaler/io/ymq/tcp_server.cpp +410 -0
  111. scaler/io/ymq/tcp_server.h +79 -0
  112. scaler/io/ymq/third_party/concurrentqueue.h +3747 -0
  113. scaler/io/ymq/timed_queue.h +272 -0
  114. scaler/io/ymq/timestamp.h +102 -0
  115. scaler/io/ymq/typedefs.h +20 -0
  116. scaler/io/ymq/utils.h +34 -0
  117. scaler/io/ymq/ymq.py +130 -0
  118. scaler/object_storage/CMakeLists.txt +50 -0
  119. scaler/object_storage/__init__.py +0 -0
  120. scaler/object_storage/constants.h +11 -0
  121. scaler/object_storage/defs.h +14 -0
  122. scaler/object_storage/io_helper.cpp +44 -0
  123. scaler/object_storage/io_helper.h +9 -0
  124. scaler/object_storage/message.cpp +56 -0
  125. scaler/object_storage/message.h +130 -0
  126. scaler/object_storage/object_manager.cpp +126 -0
  127. scaler/object_storage/object_manager.h +52 -0
  128. scaler/object_storage/object_storage_server.cpp +359 -0
  129. scaler/object_storage/object_storage_server.h +126 -0
  130. scaler/object_storage/object_storage_server.so +0 -0
  131. scaler/object_storage/pymod_object_storage_server.cpp +104 -0
  132. scaler/protocol/__init__.py +0 -0
  133. scaler/protocol/capnp/__init__.py +0 -0
  134. scaler/protocol/capnp/_python.py +6 -0
  135. scaler/protocol/capnp/common.capnp +63 -0
  136. scaler/protocol/capnp/message.capnp +216 -0
  137. scaler/protocol/capnp/object_storage.capnp +52 -0
  138. scaler/protocol/capnp/status.capnp +73 -0
  139. scaler/protocol/introduction.md +105 -0
  140. scaler/protocol/python/__init__.py +0 -0
  141. scaler/protocol/python/common.py +135 -0
  142. scaler/protocol/python/message.py +726 -0
  143. scaler/protocol/python/mixins.py +13 -0
  144. scaler/protocol/python/object_storage.py +118 -0
  145. scaler/protocol/python/status.py +279 -0
  146. scaler/protocol/worker.md +228 -0
  147. scaler/scheduler/__init__.py +0 -0
  148. scaler/scheduler/allocate_policy/__init__.py +0 -0
  149. scaler/scheduler/allocate_policy/allocate_policy.py +9 -0
  150. scaler/scheduler/allocate_policy/capability_allocate_policy.py +280 -0
  151. scaler/scheduler/allocate_policy/even_load_allocate_policy.py +159 -0
  152. scaler/scheduler/allocate_policy/mixins.py +55 -0
  153. scaler/scheduler/controllers/__init__.py +0 -0
  154. scaler/scheduler/controllers/balance_controller.py +65 -0
  155. scaler/scheduler/controllers/client_controller.py +131 -0
  156. scaler/scheduler/controllers/config_controller.py +31 -0
  157. scaler/scheduler/controllers/graph_controller.py +424 -0
  158. scaler/scheduler/controllers/information_controller.py +81 -0
  159. scaler/scheduler/controllers/mixins.py +201 -0
  160. scaler/scheduler/controllers/object_controller.py +147 -0
  161. scaler/scheduler/controllers/scaling_controller.py +86 -0
  162. scaler/scheduler/controllers/task_controller.py +373 -0
  163. scaler/scheduler/controllers/worker_controller.py +168 -0
  164. scaler/scheduler/object_usage/__init__.py +0 -0
  165. scaler/scheduler/object_usage/object_tracker.py +131 -0
  166. scaler/scheduler/scheduler.py +253 -0
  167. scaler/scheduler/task/__init__.py +0 -0
  168. scaler/scheduler/task/task_state_machine.py +92 -0
  169. scaler/scheduler/task/task_state_manager.py +61 -0
  170. scaler/ui/__init__.py +0 -0
  171. scaler/ui/constants.py +9 -0
  172. scaler/ui/live_display.py +118 -0
  173. scaler/ui/memory_window.py +146 -0
  174. scaler/ui/setting_page.py +47 -0
  175. scaler/ui/task_graph.py +370 -0
  176. scaler/ui/task_log.py +83 -0
  177. scaler/ui/utility.py +35 -0
  178. scaler/ui/webui.py +125 -0
  179. scaler/ui/worker_processors.py +85 -0
  180. scaler/utility/__init__.py +0 -0
  181. scaler/utility/debug.py +19 -0
  182. scaler/utility/event_list.py +63 -0
  183. scaler/utility/event_loop.py +58 -0
  184. scaler/utility/exceptions.py +42 -0
  185. scaler/utility/formatter.py +44 -0
  186. scaler/utility/graph/__init__.py +0 -0
  187. scaler/utility/graph/optimization.py +27 -0
  188. scaler/utility/graph/topological_sorter.py +11 -0
  189. scaler/utility/graph/topological_sorter_graphblas.py +174 -0
  190. scaler/utility/identifiers.py +105 -0
  191. scaler/utility/logging/__init__.py +0 -0
  192. scaler/utility/logging/decorators.py +25 -0
  193. scaler/utility/logging/scoped_logger.py +33 -0
  194. scaler/utility/logging/utility.py +183 -0
  195. scaler/utility/many_to_many_dict.py +123 -0
  196. scaler/utility/metadata/__init__.py +0 -0
  197. scaler/utility/metadata/profile_result.py +31 -0
  198. scaler/utility/metadata/task_flags.py +30 -0
  199. scaler/utility/mixins.py +13 -0
  200. scaler/utility/network_util.py +7 -0
  201. scaler/utility/one_to_many_dict.py +72 -0
  202. scaler/utility/queues/__init__.py +0 -0
  203. scaler/utility/queues/async_indexed_queue.py +37 -0
  204. scaler/utility/queues/async_priority_queue.py +70 -0
  205. scaler/utility/queues/async_sorted_priority_queue.py +45 -0
  206. scaler/utility/queues/indexed_queue.py +114 -0
  207. scaler/utility/serialization.py +9 -0
  208. scaler/version.txt +1 -0
  209. scaler/worker/__init__.py +0 -0
  210. scaler/worker/agent/__init__.py +0 -0
  211. scaler/worker/agent/heartbeat_manager.py +107 -0
  212. scaler/worker/agent/mixins.py +137 -0
  213. scaler/worker/agent/processor/__init__.py +0 -0
  214. scaler/worker/agent/processor/object_cache.py +107 -0
  215. scaler/worker/agent/processor/processor.py +279 -0
  216. scaler/worker/agent/processor/streaming_buffer.py +28 -0
  217. scaler/worker/agent/processor_holder.py +145 -0
  218. scaler/worker/agent/processor_manager.py +365 -0
  219. scaler/worker/agent/profiling_manager.py +109 -0
  220. scaler/worker/agent/task_manager.py +150 -0
  221. scaler/worker/agent/timeout_manager.py +19 -0
  222. scaler/worker/preload.py +84 -0
  223. scaler/worker/worker.py +264 -0
  224. scaler/worker_adapter/__init__.py +0 -0
  225. scaler/worker_adapter/native.py +154 -0
  226. scaler/worker_adapter/symphony/__init__.py +0 -0
  227. scaler/worker_adapter/symphony/callback.py +45 -0
  228. scaler/worker_adapter/symphony/heartbeat_manager.py +79 -0
  229. scaler/worker_adapter/symphony/message.py +24 -0
  230. scaler/worker_adapter/symphony/task_manager.py +288 -0
  231. scaler/worker_adapter/symphony/worker.py +205 -0
  232. scaler/worker_adapter/symphony/worker_adapter.py +142 -0
@@ -0,0 +1,729 @@
1
+ Metadata-Version: 2.2
2
+ Name: opengris-scaler
3
+ Version: 1.12.7
4
+ Summary: OpenGRIS Scaler Distribution Framework
5
+ Author-Email: Citi <opensource@citi.com>
6
+ License: Apache 2.0
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: License :: OSI Approved :: Apache Software License
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Topic :: System :: Distributed Computing
12
+ Project-URL: Home, https://github.com/Citi/scaler
13
+ Requires-Python: >=3.8
14
+ Requires-Dist: bidict
15
+ Requires-Dist: cloudpickle
16
+ Requires-Dist: psutil==7.1.0
17
+ Requires-Dist: pycapnp==2.1.0
18
+ Requires-Dist: pyzmq
19
+ Requires-Dist: sortedcontainers==2.4.0
20
+ Requires-Dist: tblib
21
+ Requires-Dist: aiohttp
22
+ Requires-Dist: graphlib-backport; python_version < "3.9"
23
+ Requires-Dist: typing-extensions>=4.0; python_version < "3.10"
24
+ Provides-Extra: uvloop
25
+ Requires-Dist: uvloop; platform_system != "Windows" and extra == "uvloop"
26
+ Provides-Extra: gui
27
+ Requires-Dist: nicegui[plotly]==2.24.2; python_version == "3.8" and extra == "gui"
28
+ Requires-Dist: nicegui[plotly]==3.0.3; python_version >= "3.9" and extra == "gui"
29
+ Provides-Extra: graphblas
30
+ Requires-Dist: python-graphblas; extra == "graphblas"
31
+ Requires-Dist: numpy==1.24.4; python_version == "3.8" and extra == "graphblas"
32
+ Requires-Dist: numpy==2.0.2; python_version == "3.9" and extra == "graphblas"
33
+ Requires-Dist: numpy==2.2.6; python_version >= "3.10" and extra == "graphblas"
34
+ Provides-Extra: all
35
+ Requires-Dist: nicegui[plotly]==2.24.2; python_version == "3.8" and extra == "all"
36
+ Requires-Dist: nicegui[plotly]==3.0.3; python_version >= "3.9" and extra == "all"
37
+ Requires-Dist: python-graphblas; extra == "all"
38
+ Requires-Dist: numpy==1.24.4; python_version == "3.8" and extra == "all"
39
+ Requires-Dist: numpy==2.0.2; python_version == "3.9" and extra == "all"
40
+ Requires-Dist: numpy==2.2.6; python_version >= "3.10" and extra == "all"
41
+ Requires-Dist: uvloop; platform_system != "Windows" and extra == "all"
42
+ Description-Content-Type: text/markdown
43
+
44
+ <div align="center">
45
+ <a href="https://github.com/finos/opengris-scaler">
46
+ <img src="https://github.com/finos/branding/blob/master/project-logos/active-project-logos/OpenGRIS/Scaler/2025_OpenGRIS_Scaler.svg" alt="OpenGRIS Scaler" width="180" height="80">
47
+ </a>
48
+
49
+ <p align="center">
50
+ Efficient, lightweight, and reliable distributed computation engine.
51
+ </p>
52
+
53
+ <p align="center">
54
+ <a href="https://community.finos.org/docs/governance/Software-Projects/stages/incubating">
55
+ <img src="https://cdn.jsdelivr.net/gh/finos/contrib-toolbox@master/images/badge-incubating.svg">
56
+ </a>
57
+ <a href="https://finos.github.io/opengris-scaler/">
58
+ <img src="https://img.shields.io/badge/Documentation-0f1632">
59
+ </a>
60
+ <a href="./LICENSE">
61
+ <img src="https://img.shields.io/github/license/citi/scaler?label=license&colorA=0f1632&colorB=255be3">
62
+ </a>
63
+ <a href="https://pypi.org/project/scaler">
64
+ <img alt="PyPI - Version" src="https://img.shields.io/pypi/v/scaler?colorA=0f1632&colorB=255be3">
65
+ </a>
66
+ <img src="https://api.securityscorecards.dev/projects/github.com/Citi/scaler/badge">
67
+ </p>
68
+ </div>
69
+
70
+ <br />
71
+
72
+ **OpenGRIS Scaler provides a simple, efficient, and reliable way to perform distributed computing** using a centralized
73
+ scheduler,
74
+ with a stable and language-agnostic protocol for client and worker communications.
75
+
76
+ ```python
77
+ import math
78
+ from scaler import Client
79
+
80
+ with Client(address="tcp://127.0.0.1:2345") as client:
81
+ # Compute a single task using `.submit()`
82
+ future = client.submit(math.sqrt, 16)
83
+ print(future.result()) # 4
84
+
85
+ # Submit multiple tasks with `.map()`
86
+ results = client.map(math.sqrt, [(i,) for i in range(100)])
87
+ print(sum(results)) # 661.46
88
+ ```
89
+
90
+ OpenGRIS Scaler is a suitable Dask replacement, offering significantly better scheduling performance for jobs with a
91
+ large number
92
+ of lightweight tasks while improving on load balancing, messaging, and deadlocks.
93
+
94
+ ## Features
95
+
96
+ - Distributed computing across **multiple cores and multiple servers**
97
+ - **Python** reference implementation, with **language-agnostic messaging protocol** built on top of
98
+ [Cap'n Proto](https://capnproto.org/) and [ZeroMQ](https://zeromq.org)
99
+ - **Graph** scheduling, which supports [Dask](https://www.dask.org)-like graph computing, with
100
+ optional [GraphBLAS](https://graphblas.org)
101
+ support for very large graph tasks
102
+ - **Automated load balancing**, which automatically balances load from busy workers to idle workers, ensuring uniform
103
+ utilization across workers
104
+ - **Automated task recovery** from worker-related hardware, OS, or network failures
105
+ - Support for **nested tasks**, allowing tasks to submit new tasks
106
+ - `top`-like **monitoring tools**
107
+ - GUI monitoring tool
108
+
109
+ ## Installation
110
+
111
+ Scaler is available on PyPI and can be installed using any compatible package manager.
112
+
113
+ ```bash
114
+ $ pip install opengris-scaler
115
+
116
+ # or with graphblas and uvloop and webui support
117
+ $ pip install opengris-scaler[graphblas,uvloop,gui]
118
+
119
+ # or simply
120
+ $ pip install opengris-scaler[all]
121
+ ```
122
+
123
+ ## Quick Start
124
+
125
+ The official documentation is available at [finos.github.io/opengris-scaler/](https://finos.github.io/opengris-scaler/).
126
+
127
+ Scaler has 3 main components:
128
+
129
+ - A **scheduler**, responsible for routing tasks to available computing resources.
130
+ - An **object storage server** that stores the task data objects (task arguments and task results).
131
+ - A set of **workers** that form a _cluster_. Workers are independent computing units, each capable of executing a
132
+ single task.
133
+ - **Clients** running inside applications, responsible for submitting tasks to the scheduler.
134
+
135
+ Please be noted that **Clients** are cross platform, supporting Windows and GNU/Linux, while other components can only
136
+ be run on GNU/Linux.
137
+
138
+ ### Start local scheduler and cluster programmatically in code
139
+
140
+ A local scheduler and a local set of workers can be conveniently started using `SchedulerClusterCombo`:
141
+
142
+ ```python
143
+ from scaler import SchedulerClusterCombo
144
+
145
+ cluster = SchedulerClusterCombo(address="tcp://127.0.0.1:2345", n_workers=4)
146
+
147
+ ...
148
+
149
+ cluster.shutdown()
150
+ ```
151
+
152
+ This will start a scheduler with 4 workers on port `2345`.
153
+
154
+ ### Setting up a computing cluster from the CLI
155
+
156
+ The object storage server, scheduler and workers can also be started from the command line with
157
+ `scaler_scheduler` and `scaler_cluster`.
158
+
159
+ First, start the scheduler, and make it connect to the object storage server:
160
+
161
+ ```bash
162
+ $ scaler_scheduler "tcp://127.0.0.1:2345"
163
+ [INFO]2025-06-06 13:13:15+0200: logging to ('/dev/stdout',)
164
+ [INFO]2025-06-06 13:13:15+0200: use event loop: builtin
165
+ [INFO]2025-06-06 13:13:15+0200: Scheduler: listen to scheduler address tcp://127.0.0.1:2345
166
+ [INFO]2025-06-06 13:13:15+0200: Scheduler: connect to object storage server tcp://127.0.0.1:2346
167
+ [INFO]2025-06-06 13:13:15+0200: Scheduler: listen to scheduler monitor address tcp://127.0.0.1:2347
168
+ ...
169
+ ```
170
+
171
+ Finally, start a set of workers (a.k.a. a Scaler *cluster*) that connects to the previously started scheduler:
172
+
173
+ ```bash
174
+ $ scaler_cluster -n 4 tcp://127.0.0.1:2345
175
+ [INFO]2023-03-19 12:19:19-0400: logging to ('/dev/stdout',)
176
+ [INFO]2023-03-19 12:19:19-0400: ClusterProcess: starting 4 workers, heartbeat_interval_seconds=2, object_retention_seconds=3600
177
+ [INFO]2023-03-19 12:19:19-0400: Worker[0] started
178
+ [INFO]2023-03-19 12:19:19-0400: Worker[1] started
179
+ [INFO]2023-03-19 12:19:19-0400: Worker[2] started
180
+ [INFO]2023-03-19 12:19:19-0400: Worker[3] started
181
+ ...
182
+ ```
183
+
184
+ Multiple Scaler clusters can be connected to the same scheduler, providing distributed computation over multiple
185
+ servers.
186
+
187
+ `-h` lists the available options for the object storage server, scheduler and the cluster executables:
188
+
189
+ ```bash
190
+ $ scaler_object_storage_server -h
191
+ $ scaler_scheduler -h
192
+ $ scaler_cluster -h
193
+ ```
194
+
195
+ ### Submitting Python tasks using the Scaler client
196
+
197
+ Knowing the scheduler address, you can connect and submit tasks from a client in your Python code:
198
+
199
+ ```python
200
+ from scaler import Client
201
+
202
+
203
+ def square(value: int):
204
+ return value * value
205
+
206
+
207
+ with Client(address="tcp://127.0.0.1:2345") as client:
208
+ future = client.submit(square, 4) # submits a single task
209
+ print(future.result()) # 16
210
+ ```
211
+
212
+ `Client.submit()` returns a standard Python future.
213
+
214
+ ## Graph computations
215
+
216
+ Scaler also supports graph tasks, for example:
217
+
218
+ ```python
219
+ from scaler import Client
220
+
221
+
222
+ def inc(i):
223
+ return i + 1
224
+
225
+
226
+ def add(a, b):
227
+ return a + b
228
+
229
+
230
+ def minus(a, b):
231
+ return a - b
232
+
233
+
234
+ graph = {
235
+ "a": 2,
236
+ "b": 2,
237
+
238
+ # the input to task c is the output of task a
239
+ "c": (inc, "a"), # c = a + 1 = 2 + 1 = 3
240
+ "d": (add, "a", "b"), # d = a + b = 2 + 2 = 4
241
+ "e": (minus, "d", "c") # e = d - c = 4 - 3 = 1
242
+ }
243
+
244
+ with Client(address="tcp://127.0.0.1:2345") as client:
245
+ result = client.get(graph, keys=["e"])
246
+ print(result) # {"e": 1}
247
+ ```
248
+
249
+ ## Configuring with TOML Files
250
+
251
+ While all Scaler components can be configured using command-line flags, using TOML files is the recommended approach for
252
+ production or shareable setups. Configuration files make your setup explicit, easier to manage, and allow you to check
253
+ your infrastructure's configuration into version control.
254
+
255
+ For convenience, you can define the settings for all components in a single, sectioned TOML file. Each component
256
+ automatically loads its configuration from its corresponding section.
257
+
258
+ ### Core Concepts
259
+
260
+ * **Usage**: To use a configuration file, pass its path via the `--config` or `-c` flag.
261
+
262
+ ```bash
263
+ scaler_scheduler --config /path/to/your/example_config.toml
264
+ ```
265
+
266
+ * **Precedence**: Settings are loaded in a specific order, with later sources overriding earlier ones. The hierarchy is:
267
+
268
+ Command-Line Flags > TOML File Settings > Built-in Default Values
269
+
270
+ * **Naming Convention**: The keys in the TOML file must match the long-form command-line arguments. The rule is to
271
+ replace any hyphens (`-`) with underscores (`_`).
272
+ * For example, the flag `--num-of-workers` becomes the TOML key `num_of_workers`.
273
+ * One can discover all available keys by running any command with the `-h` or `--help` flag.
274
+
275
+ ### Supported Components and Section Names
276
+
277
+ The following table maps each Scaler command to its corresponding section name in the TOML file.
278
+
279
+ | Command | TOML Section Name |
280
+ |----------------------------------|-----------------------------|
281
+ | `scaler_scheduler` | `[scheduler]` |
282
+ | `scaler_cluster` | `[cluster]` |
283
+ | `scaler_object_storage_server` | `[object_storage_server]` |
284
+ | `scaler_ui` | `[webui]` |
285
+ | `scaler_top` | `[top]` |
286
+ | `scaler_worker_adapter_native` | `[native_worker_adapter]` |
287
+ | `scaler_worker_adapter_symphony` | `[symphony_worker_adapter]` |
288
+
289
+ ### Practical Scenarios & Examples
290
+
291
+ #### Scenario 1: Unified Configuration File
292
+
293
+ Here is an example of a single `example_config.toml` file that configures multiple components using sections.
294
+
295
+ **example_config.toml**
296
+
297
+ ```toml
298
+ # This is a unified configuration file for all Scaler components.
299
+
300
+ [scheduler]
301
+ scheduler_address = "tcp://127.0.0.1:6378"
302
+ object_storage_address = "tcp://127.0.0.1:6379"
303
+ monitor_address = "tcp://127.0.0.1:6380"
304
+ allocate_policy = "even"
305
+ logging_level = "INFO"
306
+ logging_paths = ["/dev/stdout", "/var/log/scaler/scheduler.log"]
307
+
308
+ [cluster]
309
+ scheduler_address = "tcp://127.0.0.1:6378"
310
+ num_of_workers = 8
311
+ per_worker_capabilities = "linux,cpu=8"
312
+ task_timeout_seconds = 600
313
+
314
+ [object_storage_server]
315
+ object_storage_address = "tcp://127.0.0.1:6379"
316
+
317
+ [webui]
318
+ monitor_address = "tcp://127.0.0.1:6380"
319
+ web_port = 8081
320
+ ```
321
+
322
+ With this single file, starting your entire stack is simple and consistent:
323
+
324
+ ```bash
325
+ scaler_object_storage_server --config example_config.toml &
326
+ scaler_scheduler --config example_config.toml &
327
+ scaler_cluster --config example_config.toml &
328
+ scaler_ui --config example_config.toml &
329
+ ```
330
+
331
+ #### Scenario 2: Overriding a Section's Setting
332
+
333
+ You can override any value from the TOML file by providing it as a command-line flag. For example, to use the
334
+ example_config.toml file but test the cluster with 12 workers instead of 8:
335
+
336
+ ```bash
337
+ # The --num-of-workers flag will take precedence over the [cluster] section
338
+ scaler_cluster --config example_config.toml --num-of-workers 12
339
+ ```
340
+
341
+ The cluster will start with 12 workers, but all other settings (like `scheduler_address`) will still be loaded from the
342
+ `[cluster]` section of example_config.toml.
343
+
344
+ ## Nested computations
345
+
346
+ Scaler allows tasks to submit new tasks while being executed. Scaler also supports recursive task calls.
347
+
348
+ ```python
349
+ from scaler import Client
350
+
351
+
352
+ def fibonacci(client: Client, n: int):
353
+ if n == 0:
354
+ return 0
355
+ elif n == 1:
356
+ return 1
357
+ else:
358
+ a = client.submit(fibonacci, client, n - 1)
359
+ b = client.submit(fibonacci, client, n - 2)
360
+ return a.result() + b.result()
361
+
362
+
363
+ with Client(address="tcp://127.0.0.1:2345") as client:
364
+ future = client.submit(fibonacci, client, 8)
365
+ print(future.result()) # 21
366
+ ```
367
+
368
+ ## Task Routing and Capability Management
369
+
370
+ > **Note**: This feature is experimental and may change in future releases.
371
+
372
+ Scaler provides a task routing mechanism, allowing you to specify capability requirements for tasks and allocate them to
373
+ workers supporting these.
374
+
375
+ ### Starting the Scheduler with the Capability Allocation Policy
376
+
377
+ The scheduler can be started with the experimental capability allocation policy using the `--allocate-policy/-ap`
378
+ argument.
379
+
380
+ ```bash
381
+ $ scaler_scheduler --allocate-policy capability tcp://127.0.0.1:2345
382
+ ```
383
+
384
+ ### Defining Worker Supported Capabilities
385
+
386
+ When starting a cluster of workers, you can define the capabilities available on each worker using the
387
+ `--per-worker-capabilities/-pwc` argument. This allows the scheduler to allocate tasks to workers based on the
388
+ capabilities these provide.
389
+
390
+ ```bash
391
+ $ scaler_cluster -n 4 --per-worker-capabilities "gpu,linux" tcp://127.0.0.1:2345
392
+ ```
393
+
394
+ ### Specifying Capability Requirements for Tasks
395
+
396
+ When submitting tasks using the Scaler client, you can specify the capability requirements for each task using the
397
+ `capabilities` argument in the `submit_verbose()` and `get()` methods. This ensures that tasks are allocated to workers
398
+ supporting these capabilities.
399
+
400
+ ```python
401
+ from scaler import Client
402
+
403
+ with Client(address="tcp://127.0.0.1:2345") as client:
404
+ future = client.submit_verbose(round, args=(3.15,), kwargs={}, capabilities={"gpu": -1})
405
+ print(future.result()) # 3
406
+ ```
407
+
408
+ The scheduler will route a task to a worker if `task.capabilities.is_subset(worker.capabilities)`.
409
+
410
+ Integer values specified for capabilities (e.g., `gpu=10`) are *currently* ignored by the capabilities allocation
411
+ policy.
412
+ This means that the presence of a capabilities is considered, but not its quantity. Support for capabilities tracking
413
+ might be added in the future.
414
+
415
+ ## IBM Spectrum Symphony integration
416
+
417
+ A Scaler scheduler can interface with IBM Spectrum Symphony to provide distributed computing across Symphony clusters.
418
+
419
+ ```bash
420
+ $ scaler_worker_adapter_symphony tcp://127.0.0.1:2345 --service-name ScalerService --base-concurrency 4 --host 127.0.0.1 --port 8080
421
+ ```
422
+
423
+ This will start a Scaler worker that connects to the Scaler scheduler at `tcp://127.0.0.1:2345` and uses the Symphony
424
+ service `ScalerService` to submit tasks.
425
+
426
+ ### Symphony service
427
+
428
+ A service must be deployed in Symphony to handle the task submission.
429
+
430
+ <details>
431
+
432
+ <summary>Here is an example of a service that can be used</summary>
433
+
434
+ ```python
435
+ class Message(soamapi.Message):
436
+ def __init__(self, payload: bytes = b""):
437
+ self.__payload = payload
438
+
439
+ def set_payload(self, payload: bytes):
440
+ self.__payload = payload
441
+
442
+ def get_payload(self) -> bytes:
443
+ return self.__payload
444
+
445
+ def on_serialize(self, stream):
446
+ payload_array = array.array("b", self.get_payload())
447
+ stream.write_byte_array(payload_array, 0, len(payload_array))
448
+
449
+ def on_deserialize(self, stream):
450
+ self.set_payload(stream.read_byte_array("b"))
451
+
452
+
453
+ class ServiceContainer(soamapi.ServiceContainer):
454
+ def on_create_service(self, service_context):
455
+ return
456
+
457
+ def on_session_enter(self, session_context):
458
+ return
459
+
460
+ def on_invoke(self, task_context):
461
+ input_message = Message()
462
+ task_context.populate_task_input(input_message)
463
+
464
+ fn, *args = cloudpickle.loads(input_message.get_payload())
465
+ output_payload = cloudpickle.dumps(fn(*args))
466
+
467
+ output_message = Message(output_payload)
468
+ task_context.set_task_output(output_message)
469
+
470
+ def on_session_leave(self):
471
+ return
472
+
473
+ def on_destroy_service(self):
474
+ return
475
+ ```
476
+
477
+ </details>
478
+
479
+ ### Nested tasks
480
+
481
+ Nested task originating from Symphony workers must be able to reach the Scaler scheduler. This might require
482
+ modifications to the network configuration.
483
+
484
+ Nested tasks can also have unpredictable resource usage and runtimes, which can cause Symphony to prematurely kill
485
+ tasks. It is recommended to be conservative when provisioning resources and limits, and monitor the cluster status
486
+ closely for any abnormalities.
487
+
488
+ ### Base concurrency
489
+
490
+ Base concurrency is the maximum number of unnested tasks that can be executed concurrently. It is possible to surpass
491
+ this limit by submitting nested tasks which carry a higher priority. **Important**: If your workload contains nested
492
+ tasks the base concurrency should be set to a value less to the number of cores available on the Symphony worker or else
493
+ deadlocks may occur.
494
+
495
+ A good heuristic for setting the base concurrency is to use the following formula:
496
+
497
+ ```
498
+ base_concurrency = number_of_cores - deepest_nesting_level
499
+ ```
500
+
501
+ where `deepest_nesting_level` is the deepest nesting level a task has in your workload. For instance, if you have a
502
+ workload that has
503
+ a base task that calls a nested task that calls another nested task, then the deepest nesting level is 2.
504
+
505
+ ## Worker Adapter usage
506
+
507
+ > **Note**: This feature is experimental and may change in future releases.
508
+
509
+ Scaler provides a Worker Adapter webhook interface to integrate with other job schedulers or resource managers. The
510
+ Worker Adapter allows external systems to request the creation and termination of Scaler workers dynamically.
511
+
512
+ Please check the OpenGRIS standard for more details on the Worker Adapter
513
+ specification [here](https://github.com/finos/opengris).
514
+
515
+ ### Starting the Native Worker Adapter
516
+
517
+ Starting a Native Worker Adapter server at `http://127.0.0.1:8080`:
518
+
519
+ ```bash
520
+ $ scaler_worker_adapter_native tcp://127.0.0.1:2345 --host 127.0.0.1 --port 8080
521
+ ```
522
+
523
+ Pass the `--adapter-webhook-url` argument to the Scaler scheduler to connect to the Worker Adapter:
524
+
525
+ ```bash
526
+ $ scaler_scheduler tcp://127.0.0.1:2345 --adapter-webhook-url http://127.0.0.1:8080
527
+ ````
528
+
529
+ To check that the Worker Adapter is working, you can bring up `scaler_top` to see workers spawning and terminating as
530
+ there is task load changes.
531
+
532
+ ## Performance
533
+
534
+ ### uvloop
535
+
536
+ By default, Scaler uses Python's built-in `asyncio` event loop.
537
+ For better async performance, you can install uvloop (`pip install uvloop`) and supply `uvloop` for the CLI argument
538
+ `--event-loop` or as a keyword argument for `event_loop` in Python code when initializing the scheduler.
539
+
540
+ ```bash
541
+ scaler_scheduler --event-loop uvloop tcp://127.0.0.1:2345
542
+ ```
543
+
544
+ ```python
545
+ from scaler import SchedulerClusterCombo
546
+
547
+ scheduler = SchedulerClusterCombo(address="tcp://127.0.0.1:2345", event_loop="uvloop", n_workers=4)
548
+ ```
549
+
550
+ ## Monitoring
551
+
552
+ ### From the CLI
553
+
554
+ Use `scaler_top` to connect to the scheduler's monitor address (printed by the scheduler on startup) to see
555
+ diagnostics/metrics information about the scheduler and its workers.
556
+
557
+ ```bash
558
+ $ scaler_top tcp://127.0.0.1:2347
559
+ ```
560
+
561
+ It will look similar to `top`, but provides information about the current Scaler setup:
562
+
563
+ ```bash
564
+ scheduler | task_manager | scheduler_sent | scheduler_received
565
+ cpu 0.0% | unassigned 0 | ObjectResponse 24 | Heartbeat 183,109
566
+ rss 37.1 MiB | running 0 | TaskEcho 200,000 | ObjectRequest 24
567
+ | success 200,000 | Task 200,000 | Task 200,000
568
+ | failed 0 | TaskResult 200,000 | TaskResult 200,000
569
+ | canceled 0 | BalanceRequest 4 | BalanceResponse 4
570
+ --------------------------------------------------------------------------------------------------
571
+ Shortcuts: worker[n] cpu[c] rss[m] free[f] working[w] queued[q]
572
+
573
+ Total 10 worker(s)
574
+ worker agt_cpu agt_rss [cpu] rss free sent queued | object_id_to_tasks
575
+ W|Linux|15940|3c9409c0+ 0.0% 32.7m 0.0% 28.4m 1000 0 0 |
576
+ W|Linux|15946|d6450641+ 0.0% 30.7m 0.0% 28.2m 1000 0 0 |
577
+ W|Linux|15942|3ed56e89+ 0.0% 34.8m 0.0% 30.4m 1000 0 0 |
578
+ W|Linux|15944|6e7d5b99+ 0.0% 30.8m 0.0% 28.2m 1000 0 0 |
579
+ W|Linux|15945|33106447+ 0.0% 31.1m 0.0% 28.1m 1000 0 0 |
580
+ W|Linux|15937|b031ce9a+ 0.0% 31.0m 0.0% 30.3m 1000 0 0 |
581
+ W|Linux|15941|c4dcc2f3+ 0.0% 30.5m 0.0% 28.2m 1000 0 0 |
582
+ W|Linux|15939|e1ab4340+ 0.0% 31.0m 0.0% 28.1m 1000 0 0 |
583
+ W|Linux|15938|ed582770+ 0.0% 31.1m 0.0% 28.1m 1000 0 0 |
584
+ W|Linux|15943|a7fe8b5e+ 0.0% 30.7m 0.0% 28.3m 1000 0 0 |
585
+ ```
586
+
587
+ - **scheduler** section shows scheduler resource usage
588
+ - **task_manager** section shows count for each task status
589
+ - **scheduler_sent** section shows count for each type of messages scheduler sent
590
+ - **scheduler_received** section shows count for each type of messages scheduler received
591
+ - **function_id_to_tasks** section shows task count for each function used
592
+ - **worker** section shows worker details, , you can use shortcuts to sort by columns, and the * in the column header
593
+ shows
594
+ which column is being used for sorting
595
+ - `agt_cpu/agt_rss` means cpu/memory usage of worker agent
596
+ - `cpu/rss` means cpu/memory usage of worker
597
+ - `free` means number of free task slots for this worker
598
+ - `sent` means how many tasks scheduler sent to the worker
599
+ - `queued` means how many tasks worker received and queued
600
+
601
+ ### From the web UI
602
+
603
+ `scaler_ui` provides a web monitoring interface for Scaler.
604
+
605
+ ```bash
606
+ $ scaler_ui tcp://127.0.0.1:2347 --port 8081
607
+ ```
608
+
609
+ This will open a web server on port `8081`.
610
+
611
+ ## Slides and presentations
612
+
613
+ We showcased Scaler at FOSDEM 2025. Check out the slides
614
+ [here](<slides/Effortless Distributed Computing in Python - FOSDEM 2025.pdf>).
615
+
616
+ ## Building from source
617
+
618
+ ### Building on GNU/Linux
619
+
620
+ To contribute to Scaler, you might need to manually build its C++ components.
621
+
622
+ These C++ components depend on the Boost and Cap'n Proto libraries. If these libraries are not available on your system,
623
+ you can use the `library_tool.sh` script to download, compile, and install them (You might need `sudo`):
624
+
625
+ ```bash
626
+ ./scripts/library_tool.sh boost compile
627
+ ./scripts/library_tool.sh boost install
628
+ ./scripts/library_tool.sh capnp compile
629
+ ./scripts/library_tool.sh capnp install
630
+ ```
631
+
632
+ After installing these dependencies, use the `build.sh` script to configure, build, and install Scaler's C++ components:
633
+
634
+ ```bash
635
+ ./scripts/build.sh
636
+ ```
637
+
638
+ This script will create a build directory based on your operating system and architecture, and install the components
639
+ within the main source tree, as compiled Python modules. You can specify the compiler to use by setting the `CC` and
640
+ `CXX` environment variables.
641
+
642
+ ### Building on Windows
643
+
644
+ *Building on Windows requires _Visual Studio 17 2022_*. Similar to the former section, you can use the
645
+ `library_tool.ps1` script to download, compile, and install them (You might need `Run as administrator`):
646
+
647
+ ```bash
648
+ ./scripts/library_tool.ps1 boost compile
649
+ ./scripts/library_tool.ps1 boost install
650
+ ./scripts/library_tool.ps1 capnp compile
651
+ ./scripts/library_tool.ps1 capnp install
652
+ ```
653
+
654
+ After installing these dependencies, if you are using _Visual Studio_ for developing, you may open the project folder
655
+ with it, select preset `windows-x64`, and build the project. You may also run the following commands to configure,
656
+ build, and install Scaler's C++ components:
657
+
658
+ ```bash
659
+ cmake --preset windows-x64
660
+ cmake --build --preset windows-x64 --config (Debug|Release)
661
+ cmake --install build_windows_x64 --config (Debug|Release)
662
+ ```
663
+
664
+ The output will be similar to what described in the former section. We recommend using _Visual Studio_ for developing on
665
+ Windows.
666
+
667
+ ### Building the Python wheel
668
+
669
+ Build the Python wheel for Scaler using `cibuildwheel`:
670
+
671
+ ```bash
672
+ pip install build cibuildwheel==2.23.3
673
+
674
+ # Parametrize the cibuildwheel's container to build the Boost and Cap'n Proto dependencies.
675
+ export CIBW_BEFORE_ALL='
676
+ yum install sudo -y;
677
+ sudo ./scripts/library_tool.sh capnp compile
678
+ sudo ./scripts/library_tool.sh capnp install
679
+ sudo ./scripts/library_tool.sh boost compile
680
+ sudo ./scripts/library_tool.sh boost install'
681
+ export CIBW_BUILD="*manylinux_x86_64"
682
+ export CIBW_SKIP="pp*"
683
+ export CIBW_MANYLINUX_X86_64_IMAGE="manylinux_2_28"
684
+
685
+ python -m cibuildwheel --output-dir wheelhouse
686
+
687
+ python -m build --sdist
688
+ ```
689
+
690
+ ## Contributing
691
+
692
+ Your contributions are at the core of making this a true open source project. Any contributions you make are **greatly
693
+ appreciated**.
694
+
695
+ We welcome you to:
696
+
697
+ - Fix typos or touch up documentation
698
+ - Share your opinions on [existing issues](https://github.com/finos/opengris-scaler/issues)
699
+ - Help expand and improve our library by [opening a new issue](https://github.com/finos/opengris-scaler/issues/new)
700
+
701
+ Please review [functional contribution guidelines](./CONTRIBUTING.md) to get started 👍.
702
+
703
+ _NOTE:_ Commits and pull requests to FINOS repositories will only be accepted from those contributors with an active,
704
+ executed Individual Contributor License Agreement (ICLA) with FINOS OR contributors who are covered under an existing
705
+ and active Corporate Contribution License Agreement (CCLA) executed with FINOS. Commits from individuals not covered
706
+ under an ICLA or CCLA will be flagged and blocked by
707
+ the ([EasyCLA](https://community.finos.org/docs/governance/Software-Projects/easycla)) tool. Please note that some CCLAs
708
+ require individuals/employees to be explicitly named on the CCLA.
709
+
710
+ *Need an ICLA? Unsure if you are covered under an existing CCLA? Email [help@finos.org](mailto:help@finos.org)*
711
+
712
+ ## Code of Conduct
713
+
714
+ Please see the FINOS [Community Code of Conduct](https://www.finos.org/code-of-conduct).
715
+
716
+ ## License
717
+
718
+ Copyright 2023 Citigroup, Inc.
719
+
720
+ This project is distributed under the [Apache-2.0 License](https://www.apache.org/licenses/LICENSE-2.0). See
721
+ [`LICENSE`](./LICENSE) for more information.
722
+
723
+ SPDX-License-Identifier: [Apache-2.0](https://spdx.org/licenses/Apache-2.0)
724
+
725
+ ## Contact
726
+
727
+ If you have a query or require support with this
728
+ project, [raise an issue](https://github.com/finos/opengris-scaler/issues).
729
+ Otherwise, reach out to [opensource@citi.com](mailto:opensource@citi.com).