parsl 2024.12.23__py3-none-any.whl → 2025.1.6__py3-none-any.whl

parsl/jobs/strategy.py

@@ -298,8 +298,8 @@ class Strategy:
                     # Scale in for htex
                     if isinstance(executor, HighThroughputExecutor):
                         if active_blocks > min_blocks:
-                            excess_slots = math.ceil(active_slots - (active_tasks * parallelism))
-                            excess_blocks = math.ceil(float(excess_slots) / (tasks_per_node * nodes_per_block))
+                            excess_slots = math.floor(active_slots - (active_tasks * parallelism))
+                            excess_blocks = math.floor(float(excess_slots) / (tasks_per_node * nodes_per_block))
                             excess_blocks = min(excess_blocks, active_blocks - min_blocks)
                             logger.debug(f"Requesting scaling in by {excess_blocks} blocks with idle time {self.max_idletime}s")
                             executor.scale_in_facade(excess_blocks, max_idletime=self.max_idletime)

parsl/monitoring/db_manager.py

@@ -78,7 +78,7 @@ class Database:
 
     def _get_mapper(self, table_obj: Table) -> Mapper:
         all_mappers: Set[Mapper] = set()
-        for mapper_registry in mapperlib._all_registries():  # type: ignore[attr-defined]
+        for mapper_registry in mapperlib._all_registries():
             all_mappers.update(mapper_registry.mappers)
         mapper_gen = (
             mapper for mapper in all_mappers

parsl/tests/test_monitoring/test_app_names.py

@@ -67,7 +67,7 @@ def test_app_name(get_app, expected_name, expected_result, tmpd_cwd):
     with engine.begin() as connection:
 
         def count_rows(table: str):
-            result = connection.execute(f"SELECT COUNT(*) FROM {table}")
+            result = connection.execute(sqlalchemy.text(f"SELECT COUNT(*) FROM {table}"))
             (c, ) = result.first()
             return c
 
@@ -81,6 +81,6 @@ def test_app_name(get_app, expected_name, expected_result, tmpd_cwd):
         assert count_rows("try") == 1
 
         # ... and has the expected name.
-        result = connection.execute("SELECT task_func_name FROM task")
+        result = connection.execute(sqlalchemy.text("SELECT task_func_name FROM task"))
         (c, ) = result.first()
         assert c == expected_name

parsl/tests/test_monitoring/test_stdouterr.py

@@ -106,7 +106,7 @@ def test_stdstream_to_monitoring(stdx, expected_stdx, stream, tmpd_cwd, caplog):
     with engine.begin() as connection:
 
         def count_rows(table: str):
-            result = connection.execute(f"SELECT COUNT(*) FROM {table}")
+            result = connection.execute(sqlalchemy.text(f"SELECT COUNT(*) FROM {table}"))
             (c, ) = result.first()
             return c
 
@@ -120,7 +120,7 @@ def test_stdstream_to_monitoring(stdx, expected_stdx, stream, tmpd_cwd, caplog):
         assert count_rows("try") == 1
 
         # ... and has the expected name.
-        result = connection.execute(f"SELECT task_{stream} FROM task")
+        result = connection.execute(sqlalchemy.text(f"SELECT task_{stream} FROM task"))
         (c, ) = result.first()
 
         if isinstance(expected_stdx, str):

parsl/tests/test_scaling/test_regression_3696_oscillation.py

@@ -0,0 +1,103 @@
+import math
+from unittest.mock import MagicMock
+
+import pytest
+
+from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.jobs.states import JobState, JobStatus
+from parsl.jobs.strategy import Strategy
+
+
+# the parameterize tuple consists of:
+# Input:
+#  * number of tasks to mock the load as
+#  * number of workers per node
+# Expected output:
+#  * the number of blocks we should expect to be launched
+#    in this situation
+#
+# This test will configure an executor, then run strategize
+# a few times, asserting that it converges to the correct
+# number of blocks without oscillating.
+@pytest.mark.local
+@pytest.mark.parametrize("ns", [(14, 48, 1),    # values from issue #3696
+
+                                (1, 1, 1),      # one task needs one block
+
+                                (100, 1, 20),   # many one-task blocks, hitting hard-coded max blocks
+
+                                (47, 48, 1),    # some edge cases around #3696 values
+                                (48, 48, 1),    # "
+                                (49, 48, 2),    # "
+                                (149, 50, 3)])  # "
+def test_htex_strategy_does_not_oscillate(ns):
+    """Check for oscillations in htex scaling.
+    In issue 3696, with a large number of workers per block
+    and a smaller number of active tasks, the htex scaling
+    strategy oscillates between 0 and 1 active block, rather
+    than converging to 1 active block.
+    """
+
+    n_tasks, n_workers, n_blocks = ns
+
+    s = Strategy(strategy='htex_auto_scale', max_idletime=0)
+
+    provider = MagicMock()
+    executor = MagicMock(spec=HighThroughputExecutor)
+
+    statuses = {}
+
+    executor.provider = provider
+    executor.outstanding = n_tasks
+    executor.status_facade = statuses
+    executor.workers_per_node = n_workers
+
+    provider.parallelism = 1
+    provider.init_blocks = 0
+    provider.min_blocks = 0
+    provider.max_blocks = 20
+    provider.nodes_per_block = 1
+
+    def scale_out(n):
+        for _ in range(n):
+            statuses[len(statuses)] = JobStatus(state=JobState.PENDING)
+
+    executor.scale_out_facade.side_effect = scale_out
+
+    def scale_in(n, max_idletime=None):
+        # find n PENDING jobs and set them to CANCELLED
+        for k in statuses:
+            if n == 0:
+                return
+            if statuses[k].state == JobState.PENDING:
+                statuses[k].state = JobState.CANCELLED
+                n -= 1
+
+    executor.scale_in_facade.side_effect = scale_in
+
+    s.add_executors([executor])
+
+    # In issue #3696, this first strategise does initial and load based
+    # scale outs, because n_tasks > n_workers*0
+    s.strategize([executor])
+
+    executor.scale_out_facade.assert_called()
+    assert len(statuses) == n_blocks, "Should have launched n_blocks"
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == n_blocks
+    # there might be several calls to scale_out_facade inside strategy,
+    # but the end effect should be that exactly one block is scaled out.
+
+    executor.scale_in_facade.assert_not_called()
+
+    # In issue #3696, this second strategize does a scale in, because n_tasks < n_workers*1
+    s.strategize([executor])
+
+    # assert that there should still be n_blocks pending blocks
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == n_blocks
+    # this assert fails due to issue #3696
+
+    # Now check scale in happens with 0 load
+    executor.outstanding = 0
+    s.strategize([executor])
+    executor.scale_in_facade.assert_called()
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == 0

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.12.23'
+VERSION = '2025.01.06'

{parsl-2024.12.23.dist-info → parsl-2025.1.6.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.12.23
+Version: 2025.1.6
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.12.23.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.01.06.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -28,9 +28,9 @@ Requires-Dist: psutil>=5.5.1
 Requires-Dist: setproctitle
 Requires-Dist: filelock<4,>=3.13
 Provides-Extra: all
-Requires-Dist: sqlalchemy<2,>=1.4; extra == "all"
-Requires-Dist: pydot; extra == "all"
-Requires-Dist: networkx<2.6,>=2.5; extra == "all"
+Requires-Dist: sqlalchemy<2.1,>=2; extra == "all"
+Requires-Dist: pydot>=1.4.2; extra == "all"
+Requires-Dist: networkx<3.3,>=3.2; extra == "all"
 Requires-Dist: Flask>=1.0.2; extra == "all"
 Requires-Dist: flask-sqlalchemy; extra == "all"
 Requires-Dist: pandas<2.2; extra == "all"
@@ -76,15 +76,15 @@ Requires-Dist: python-gssapi; extra == "gssapi"
 Provides-Extra: kubernetes
 Requires-Dist: kubernetes; extra == "kubernetes"
 Provides-Extra: monitoring
-Requires-Dist: sqlalchemy<2,>=1.4; extra == "monitoring"
+Requires-Dist: sqlalchemy<2.1,>=2; extra == "monitoring"
 Provides-Extra: proxystore
 Requires-Dist: proxystore; extra == "proxystore"
 Provides-Extra: radical-pilot
 Requires-Dist: radical.pilot==1.90; extra == "radical-pilot"
 Requires-Dist: radical.utils==1.90; extra == "radical-pilot"
 Provides-Extra: visualization
-Requires-Dist: pydot; extra == "visualization"
-Requires-Dist: networkx<2.6,>=2.5; extra == "visualization"
+Requires-Dist: pydot>=1.4.2; extra == "visualization"
+Requires-Dist: networkx<3.3,>=3.2; extra == "visualization"
 Requires-Dist: Flask>=1.0.2; extra == "visualization"
 Requires-Dist: flask-sqlalchemy; extra == "visualization"
 Requires-Dist: pandas<2.2; extra == "visualization"

{parsl-2024.12.23.dist-info → parsl-2025.1.6.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=MyaEcEq-Qf860u7V98u-PZrPNdtzOZL_NW6EhIJnmfQ,1937
 parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=5FvHIMao3Ik0Rm2p2ieL1KQcQcYXc5K83Jrx5csi-B4,14301
-parsl/version.py,sha256=6l0WNaP0JU1FwTVCPx69g0nu1kKQFf-XcxlNgQVkHuo,131
+parsl/version.py,sha256=lIhZrp6NnBCcbBMgntHRdPaNhpDBWe88rZRBa8C0pSw,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=0gbM4AH2OtFOLsv07I5nglpElcwMSOi-FzdZZfrk7So,8532
 parsl/app/bash.py,sha256=jm2AvePlCT9DZR7H_4ANDWxatp5dN_22FUlT_gWhZ-g,5528
@@ -105,13 +105,13 @@ parsl/jobs/error_handlers.py,sha256=BBXwUAMJpBm0HxV1P-I6jv7ZF9wcrhnCfzSTlsd2g4w,
 parsl/jobs/errors.py,sha256=cpSQXCrlKtuHsQf7usjF-lX8XsDkFnE5kWpmFjiN6OU,178
 parsl/jobs/job_status_poller.py,sha256=b37JOqDpSesqeSreEh1HzfVTFnD5Aoy6k8JDXkkPDmk,2192
 parsl/jobs/states.py,sha256=dUM8gC4YVpUjLMARJJ_tDERs6oHsoNheAtG6JWPIJt4,5058
-parsl/jobs/strategy.py,sha256=KYcIpjWVKLYbM0TXhue9Zp2a7I1zkbHx4raPFiDlewA,13799
+parsl/jobs/strategy.py,sha256=wQk5q-K0adh1oZFI9Mei2iASLRQ8khup1ebiZD-4xfQ,13801
 parsl/launchers/__init__.py,sha256=jJeDOWGKJjvpmWTLsj1zSqce_UAhWRc_IO-TzaOAlII,579
 parsl/launchers/base.py,sha256=CblcvPTJiu-MNLWaRtFe29SZQ0BpTOlaY8CGcHdlHIE,538
 parsl/launchers/errors.py,sha256=8YMV_CHpBNVa4eXkGE4x5DaFQlZkDCRCHmBktYcY6TA,467
 parsl/launchers/launchers.py,sha256=cQsNsHuCOL_nQTjPXf0--YsgsDoMoJ77bO1Wt4ncLjs,15134
 parsl/monitoring/__init__.py,sha256=0ywNz6i0lM1xo_7_BIxhETDGeVd2C_0wwD7qgeaMR4c,83
-parsl/monitoring/db_manager.py,sha256=D8lrngFGxbFhyWVkF8JZRTbGxRYmd3SY6_zu8KV0FJs,33330
+parsl/monitoring/db_manager.py,sha256=ra5PqmbUstfDx0o_bkBYI8GIUi461-GV3b4A-Q6DVVE,33300
 parsl/monitoring/errors.py,sha256=D6jpYzEzp0d6FmVKGqhvjAxr4ztZfJX2s-aXemH9bBU,148
 parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
 parsl/monitoring/monitoring.py,sha256=fkBZU4fWp7qBQUcKYtWjd4d-SsFlJUZNMZacFOh0IoA,12687
@@ -330,14 +330,14 @@ parsl/tests/test_htex/test_resource_spec_validation.py,sha256=VzOk4rjMNiDcEVLb-3
 parsl/tests/test_htex/test_worker_failure.py,sha256=Uz-RHI-LK78FMjXUvrUFmo4iYfmpDVBUcBxxRb3UG9M,603
 parsl/tests/test_htex/test_zmq_binding.py,sha256=WNFsCKKfid2uEfem0WLgl1wnBncIabpAv6kmg3imBxk,4001
 parsl/tests/test_monitoring/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/tests/test_monitoring/test_app_names.py,sha256=ayyxySGWpKSe9dDw2UeJo1dicxjpALRuLsJfprZV4Eg,2174
+parsl/tests/test_monitoring/test_app_names.py,sha256=A-mOMCVhZDnUyJp32fsTUkHdcyval8o7WPEWacDkbD4,2208
 parsl/tests/test_monitoring/test_basic.py,sha256=VdF6JHfqsEOIMg-ysIAREgygZIjHWNDVLNVQ7jhWxmQ,4592
 parsl/tests/test_monitoring/test_db_locks.py,sha256=3s3c1xhKo230ZZIJ3f1Ca4U7LcEdXnanOGVXQyNlk2U,2895
 parsl/tests/test_monitoring/test_fuzz_zmq.py,sha256=--3-pQUvXXbkr8v_BEJoPvVvNly1oXvrD2nJh6yl_0M,3436
 parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py,sha256=_QV8zjBKVF_qBbBnhT0C3X9AmfS7IKLcOnEw_cU6HeM,2622
 parsl/tests/test_monitoring/test_incomplete_futures.py,sha256=ZnO1sFSwlWUBHX64C_zwfTVRVC_UFNlU4h0POgx6NEo,2005
 parsl/tests/test_monitoring/test_memoization_representation.py,sha256=dknv2nO7pNZ1jGxWGsC_AW3rs90gjMIeC5d7pIJ75Xc,2645
-parsl/tests/test_monitoring/test_stdouterr.py,sha256=9jlzeWU7csjPmcaC09RR6i9U8eTLdUm_R_hTGli0k58,4458
+parsl/tests/test_monitoring/test_stdouterr.py,sha256=AjzD4oumRIe9XMDPU4Cn_9-fwx34Vg8eAq58VfCO_zc,4492
 parsl/tests/test_monitoring/test_viz_colouring.py,sha256=83Qdmn3gM0j7IL6kPDcuIsp_nl4zj-liPijyIN632SY,592
 parsl/tests/test_mpi_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_mpi_apps/test_bad_mpi_config.py,sha256=QKvEUSrHIBrvqu2fRj1MAqxsYxDfcrdQ7dzWdOZejuU,1320
@@ -401,6 +401,7 @@ parsl/tests/test_scaling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 parsl/tests/test_scaling/test_block_error_handler.py,sha256=OS1IyiK8gjRFI1VzpmOvEnKsPev2vKmC6Z2Hp5LaHpA,6068
 parsl/tests/test_scaling/test_regression_1621.py,sha256=e3-bkHR3d8LxA-uY0BugyWgYzksh00I_UbaA-jHOzKY,1872
 parsl/tests/test_scaling/test_regression_3568_scaledown_vs_MISSING.py,sha256=bjE_NIBoWK6heEz5LN0tzE1977vUA9kVemAYCqcIbzY,2942
+parsl/tests/test_scaling/test_regression_3696_oscillation.py,sha256=7Xc3vgocXXUbUegh9t5OyXlV91lRXDVMUlrOwErYOXA,3621
 parsl/tests/test_scaling/test_scale_down.py,sha256=vHJOMRUriW6xPtaY8GTKYXd5P0WJkQV6Q1IPui05aLU,2736
 parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py,sha256=EnVNllKO2AGKkGa6927cLrzvvG6mpNQeFDzVktv6x08,4521
 parsl/tests/test_scaling/test_scale_down_htex_unregistered.py,sha256=OrdnYmd58n7UfkANPJ7mzha4WSCPdbgJRX1O1Zdu0tI,1954
@@ -449,13 +450,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
 parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
 parsl/usage_tracking/usage.py,sha256=tcoZ2OUjsQVakG8Uu9_HFuEdzpSHyt4JarSRcLGnSMw,8918
-parsl-2024.12.23.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
-parsl-2024.12.23.data/scripts/interchange.py,sha256=rUhF_Bwk5NOqLhh-HgP-ei_gclKnPIJJ7uS32p0j-XI,30129
-parsl-2024.12.23.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
-parsl-2024.12.23.data/scripts/process_worker_pool.py,sha256=82FoJTye2SysJzPg-N8BpenuHGU7hOI8-Bedq8HV9C0,41851
-parsl-2024.12.23.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2024.12.23.dist-info/METADATA,sha256=19V6tjgV6yT4ycIAUFZS-seRIXUzQwSUvk6rebdbAt0,3848
-parsl-2024.12.23.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-parsl-2024.12.23.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2024.12.23.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2024.12.23.dist-info/RECORD,,
+parsl-2025.1.6.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
+parsl-2025.1.6.data/scripts/interchange.py,sha256=rUhF_Bwk5NOqLhh-HgP-ei_gclKnPIJJ7uS32p0j-XI,30129
+parsl-2025.1.6.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
+parsl-2025.1.6.data/scripts/process_worker_pool.py,sha256=82FoJTye2SysJzPg-N8BpenuHGU7hOI8-Bedq8HV9C0,41851
+parsl-2025.1.6.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2025.1.6.dist-info/METADATA,sha256=vd794sFRn6CYChMUdYFIu7M5BZTvn18yg0Ymji3XGjI,3860
+parsl-2025.1.6.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+parsl-2025.1.6.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2025.1.6.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2025.1.6.dist-info/RECORD,,