pythagoras 0.20.8__py3-none-any.whl → 0.20.10__py3-none-any.whl

pythagoras/_090_swarming_portals/swarming_portals.py

@@ -132,10 +132,16 @@ class SwarmingPortal(PureCodePortal):
     @property
     def max_n_workers(self) -> int:
         """Get the maximum number of background workers"""
-        n = self._get_config_setting("max_n_workers")
-        if n in (None, KEEP_CURRENT):
-            n = 10
-        return n
+        if not hasattr(self, "_max_n_workers_cache"):
+            n = self._get_config_setting("max_n_workers")
+            if n in (None, KEEP_CURRENT):
+                n = 10
+            n = min(n, get_available_cpu_cores())
+            n = min(n, get_available_ram_mb() / 500)
+            n = int(n)
+            self._max_n_workers_cache = n
+
+        return self._max_n_workers_cache
 
 
     def describe(self) -> pd.DataFrame:
@@ -173,6 +179,11 @@ class SwarmingPortal(PureCodePortal):
             delay = self.entropy_infuser.uniform(min_delay, max_delay)
             sleep(delay)
 
+
+    def _invalidate_cache(self):
+        if hasattr(self, "_max_n_workers_cache"):
+            del self._max_n_workers_cache
+
 parameterizable.register_parameterizable_class(SwarmingPortal)
 
 

pythagoras/_100_top_level_API/__init__.py

@@ -1,2 +1,2 @@
 from .top_level_API import *
-from .default_local_portal import instantiate_default_local_portal
+from .default_local_portal import *

pythagoras/_100_top_level_API/top_level_API.py

@@ -0,0 +1,30 @@
+import os
+from urllib.parse import urlparse
+
+from .._090_swarming_portals import SwarmingPortal
+
+
+def _is_valid_url(url: str) -> bool:
+    try:
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except ValueError:
+        return False
+
+def _is_valid_folder_name(path_str):
+    if os.path.isdir(path_str):
+        return True
+    try:
+        os.makedirs(path_str, exist_ok=True)
+        return True
+    except OSError:
+        return False
+
+def get_portal(url_or_foldername:str) -> SwarmingPortal:
+    """Create a return a portal object based on a URL or a directory.   """
+    if _is_valid_folder_name(url_or_foldername):
+        return SwarmingPortal(url_or_foldername)
+    elif _is_valid_url(url_or_foldername):
+        raise NotImplementedError("URLы not supported yet ... stay turned")
+    else:
+        raise ValueError(f"Invalid folder name: {url_or_foldername}")

pythagoras/core/__init__.py

@@ -1,4 +1,5 @@
 from .._030_data_portals import ready, get
 from .._060_autonomous_code_portals import autonomous
 from .._080_pure_code_portals import pure
-from .._090_swarming_portals import SwarmingPortal
+from .._090_swarming_portals import SwarmingPortal
+from .._100_top_level_API import get_portal

{pythagoras-0.20.8.dist-info → pythagoras-0.20.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pythagoras
-Version: 0.20.8
+Version: 0.20.10
 Summary: Planet-scale distributed computing in Python.
 Keywords: cloud,ML,AI,serverless,distributed,parallel,machine-learning,deep-learning,pythagoras
 Author: Volodymyr (Vlad) Pavlov
@@ -61,7 +61,7 @@ data science, machine learning, and AI workflows.
 Pythagoras offers:
 
 * seamless parallel execution of Python code, locally or 
-in diverse distributed environments of any scale;
+in diverse distributed environments of virtually any scale;
 * ubiquitous caching for intermediate program states, 
 with intelligent data and code change tracking;
 * simple and budget-friendly orchestration backend.

{pythagoras-0.20.8.dist-info → pythagoras-0.20.10.dist-info}/RECORD

@@ -46,11 +46,11 @@ pythagoras/_080_pure_code_portals/pure_core_classes.py,sha256=73167f9aab4c91f5a0
 pythagoras/_080_pure_code_portals/pure_decorator.py,sha256=d908ad5eae50e59eeab32f738f784605708e00de71dd53dcbf25efecb83d110d,1168
 pythagoras/_090_swarming_portals/__init__.py,sha256=7041578f84ffa291f2752c7a2168007b9113f99482f0173f3729171b3bff551a,32
 pythagoras/_090_swarming_portals/output_suppressor.py,sha256=83e6cc9bcc62a226babb1165912ef5095ea948499ce5136a7516ac8b54522607,626
-pythagoras/_090_swarming_portals/swarming_portals.py,sha256=cb5240d0a070571a1b58d66fe3013c5d3123886ce4b266aeb35e95cc28d033fa,11633
+pythagoras/_090_swarming_portals/swarming_portals.py,sha256=92f0f95c25775e98bce777e007774b6dcbad27351a41f053c498c47e5519b753,12019
 pythagoras/_090_swarming_portals/system_utils.py,sha256=d125a0b705b9f8477712179c49441daaf2cd2b0b0d4494b6ac977b2df87b50b5,1370
-pythagoras/_100_top_level_API/__init__.py,sha256=edac726311978503953fe4c37c475c7d616708236880f393b63894b9ab6251e5,95
+pythagoras/_100_top_level_API/__init__.py,sha256=b392edc2c918da7c2444f14accfd0fac2cd0d5cf6849c64ed2433dfdb58b8b75,64
 pythagoras/_100_top_level_API/default_local_portal.py,sha256=cfbe20499fed2f038b507b44fb58bb4cb6ea2fbe2fe93a3ab5ad7f3ac655005f,215
-pythagoras/_100_top_level_API/top_level_API.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
+pythagoras/_100_top_level_API/top_level_API.py,sha256=4b63575b86df2fdf28e93c67e8d33411e05bd67cf016d3d297ec9635ebc04081,906
 pythagoras/_800_signatures_and_converters/__init__.py,sha256=d9420254db0c62903fe5419612cef536f219ec319e81b7efc3e783a113aac93b,167
 pythagoras/_800_signatures_and_converters/base_16_32_convertors.py,sha256=66356d2a38e33203201d484cfb543da8b6160832a1965aaaf73e1880a2f4a828,1236
 pythagoras/_800_signatures_and_converters/current_date_gmt_str.py,sha256=2beac61f2ad112cc8175832aeef5abbf715d01a71ea56429bce7d3fe6ad9a8d3,268
@@ -60,7 +60,7 @@ pythagoras/_800_signatures_and_converters/random_signatures.py,sha256=c174f28c04
 pythagoras/_900_project_stats_collector/__init__.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
 pythagoras/_900_project_stats_collector/project_analyzer.py,sha256=d06e9d7b516cb7424ef777e70abe9d5220e09b0b19476326b8974b4dc3917f89,3506
 pythagoras/__init__.py,sha256=94303c01a7bde4078fdbd90c0b142807a023fa352c473c3a544a2180b7254ae9,1062
-pythagoras/core/__init__.py,sha256=26640880921763e8801b74da28fccf1cd47ac0ba0d40ee8fcaf7d748410406bf,189
-pythagoras-0.20.8.dist-info/WHEEL,sha256=1f21b63a61110964f543b6041e8b7da078f20c09d500a32cb79eee5c3d655f54,79
-pythagoras-0.20.8.dist-info/METADATA,sha256=da94d8c4b1a7c7173f25b935fb1f941baf58b6c9cbbc500add1e91854fba1b24,4177
-pythagoras-0.20.8.dist-info/RECORD,,
+pythagoras/core/__init__.py,sha256=a6c3ec6f7d3e69221be505fd344061b774b0eb5c2f18431f39abb9426cb5d0cc,233
+pythagoras-0.20.10.dist-info/WHEEL,sha256=1f21b63a61110964f543b6041e8b7da078f20c09d500a32cb79eee5c3d655f54,79
+pythagoras-0.20.10.dist-info/METADATA,sha256=855a563ba37a03a947eafd7fed54aa24e0ff4bf7d5839ee0e9efabf459ed42f6,4188
+pythagoras-0.20.10.dist-info/RECORD,,