regscale-cli 6.25.0.1__py3-none-any.whl → 6.25.1.0__py3-none-any.whl

regscale/utils/threading/threadhandler.py

@@ -3,7 +3,7 @@
 """Thread handler to create and start threads"""
 
 from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed
-from typing import Any, Callable, List, Tuple, Union
+from typing import Any, Callable, List, Optional, Tuple, Union
 
 
 class ThreadManager:
@@ -98,16 +98,7 @@ def create_threads(process: Callable, args: Tuple, thread_count: int) -> None:
     :param int thread_count: # of threads needed
     :rtype: None
     """
-    # set max threads
-    from regscale.core.app.application import Application
-
-    app = Application()
-    max_threads = app.config.get("maxThreads", 100)
-    if not isinstance(max_threads, int):
-        try:
-            max_threads = int(max_threads)
-        except (ValueError, TypeError):
-            max_threads = 100
+    max_threads = _determine_max_threads()
     if threads := min(thread_count, max_threads):
         # start the threads with the number of threads allowed
         with ThreadPoolExecutor(max_workers=threads) as executor:
@@ -127,15 +118,29 @@ def thread_assignment(thread: int, total_items: int) -> list:
     :return: List of items to process for the given thread
     :rtype: list
     """
+    max_threads = _determine_max_threads()
+
+    return [x for x in range(total_items) if x % max_threads == thread]
+
+
+def _determine_max_threads(default: Optional[int] = 100) -> int:
+    """
+    Function to determine the max threads to use by checking the Application config
+
+    :param Optional[int] default: The default max threads to use if the Application config is not set, defaults to 100
+    :return: The max threads to use
+    :rtype: int
+    """
     from regscale.core.app.application import Application
 
     app = Application()
+    if app.running_in_airflow:
+        default = 20
     # set max threads
-    max_threads = app.config.get("maxThreads", 100)
+    max_threads = app.config.get("maxThreads", default)
     if not isinstance(max_threads, int):
         try:
             max_threads = int(max_threads)
         except (ValueError, TypeError):
-            max_threads = 100
-
-    return [x for x in range(total_items) if x % max_threads == thread]
+            max_threads = default
+    return max_threads

{regscale_cli-6.25.0.1.dist-info → regscale_cli-6.25.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: regscale-cli
-Version: 6.25.0.1
+Version: 6.25.1.0
 Summary: Command Line Interface (CLI) for bulk processing/loading data into RegScale
 Home-page: https://github.com/RegScale/regscale-cli
 Author: Travis Howerton

{regscale_cli-6.25.0.1.dist-info → regscale_cli-6.25.1.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 regscale/__init__.py,sha256=ZygAIkX6Nbjag1czWdQa-yP-GM1mBE_9ss21Xh__JFc,34
-regscale/_version.py,sha256=uOMrPCnBFPxcbCOIl5yNTm-LwhdfzpkFWbAUt-kLbvI,1198
+regscale/_version.py,sha256=qXIgTADAR4x7Bs08Sx0ILzqiq6QkTVIVUbXG1Q5wDZQ,1198
 regscale/regscale.py,sha256=4LzcbkBxPh23cCLANVEEr6uNoZ0sebHec0N-VJm5_vA,31480
 regscale/airflow/__init__.py,sha256=yMwN0Bz4JbM0nl5qY_hPegxo_O2ilhTOL9PY5Njhn-s,270
 regscale/airflow/click_dags.py,sha256=H3SUR5jkvInNMv1gu-VG-Ja_H-kH145CpQYNalWNAbE,4520
@@ -112,10 +112,10 @@ regscale/integrations/__init__.py,sha256=Sqthp3Jggo7co_go380cLn3OAb0cHwqL609_4QJ
 regscale/integrations/api_paginator.py,sha256=73rjaNM9mGv8evHAeoObXEjZPg-bJuGPo60ewCLEil8,33192
 regscale/integrations/api_paginator_example.py,sha256=lEuYI-xEGcjnXuIzbCobCP0YRuukLF0s8S3d382SAH4,12119
 regscale/integrations/compliance_integration.py,sha256=as4cb4NWmY9mTaHtu75lvwxScB3OKE3Ge6ki7-zZNBE,91103
-regscale/integrations/due_date_handler.py,sha256=qIOeFx4LPvOTpqJx8JV5Hj3FLlx0FoaYAsFoXqxYBcs,8343
+regscale/integrations/due_date_handler.py,sha256=RIT5OuKLeEU2XkSv5i8NIAyrLmJ39ZY4XFLxylw-L34,13861
 regscale/integrations/integration_override.py,sha256=HjYBCuvNpU3t3FptaqYAkdexLFOZBAFrFE9xU0RD1Nc,5867
 regscale/integrations/jsonl_scanner_integration.py,sha256=l8nq_T3rE1XX-6HxrNHm3xzxCNWbIjxQvGMdtZWs7KQ,57003
-regscale/integrations/scanner_integration.py,sha256=T9AkwDdL1V2e6G31MLFYsbMzkyX4emqmomsp_muOdew,187010
+regscale/integrations/scanner_integration.py,sha256=Owy06vAXzYnq60tXG07oMpZGjThDwsXKngBa5U5MqLg,187636
 regscale/integrations/variables.py,sha256=MfQ34WuYVN5437A9sZ2ssHRkA3gFhMHfk1DVasceGmY,2336
 regscale/integrations/commercial/__init__.py,sha256=U2RIsRe4lKufADRwMJY0RhSbL6TX4jLxpCtbcnzvI3k,14607
 regscale/integrations/commercial/ad.py,sha256=YXSmK8vRf6yi2GnREGa5GrE6GelhFrLj44SY8AO1pK0,15509
@@ -218,7 +218,7 @@ regscale/integrations/commercial/stigv2/ckl_parser.py,sha256=RPrDGsKqAbTJMsJd3AY
 regscale/integrations/commercial/stigv2/click_commands.py,sha256=-765vFF03aSjBxU9SYCpca7x8fPaoTDNQtbk4fKuwYw,2792
 regscale/integrations/commercial/stigv2/stig_integration.py,sha256=-UlfL0BBI3jFGotctOJyTCRAU9zIXW6mwU7ySrRXE9U,11573
 regscale/integrations/commercial/synqly/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-regscale/integrations/commercial/synqly/assets.py,sha256=2D26tvMvNIpR9YuD0P2h_6oqXfZf5y93xGhb2hPFsn0,6785
+regscale/integrations/commercial/synqly/assets.py,sha256=EY495sxSItKxkc5tp15KU_t8X9-MMt0-JCblv676QTY,7468
 regscale/integrations/commercial/synqly/edr.py,sha256=4EQSI2-5QnaWXh1idhndToFDOHOb3xycWAdc8NY4Auk,2836
 regscale/integrations/commercial/synqly/query_builder.py,sha256=1MAheCLG_3zUkvDeXbnCFEBybbG09OV-vUtBD0GLx0o,19306
 regscale/integrations/commercial/synqly/ticketing.py,sha256=akucc9Ok4cTcivH_rmG927WJetJDhZ0L4IF27OC_KOE,8352
@@ -241,7 +241,7 @@ regscale/integrations/commercial/wizv2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 regscale/integrations/commercial/wizv2/async_client.py,sha256=CElkHxbSlNfRnvJpf4JTBD293NJxQsR-QaTBuHINrLw,12946
 regscale/integrations/commercial/wizv2/click.py,sha256=FkoXEUgZGQUyZNMdaDJTyNE8G_oXmIM5z6JlT-QQW8I,19831
 regscale/integrations/commercial/wizv2/compliance_report.py,sha256=lFTgm78lrTSXJyqGA-wUbKJ2jOnFaHsV8_Ex95iIO-M,68717
-regscale/integrations/commercial/wizv2/constants.py,sha256=emJHsfdsRuHhWTYqADV4-l6sZNJ-nWGj_l1abdFrXxk,49308
+regscale/integrations/commercial/wizv2/constants.py,sha256=q5oxB4bMM0e45K4zQUqS_B-hHsEe8gpT3K9jrkg4_cg,48314
 regscale/integrations/commercial/wizv2/data_fetcher.py,sha256=CmJyUCeTQIkYBu2LPicpTNwXJrYkRP3xxM55BJQhr2A,16258
 regscale/integrations/commercial/wizv2/file_cleanup.py,sha256=ENBO9RDXL2kyHkBYaSai83i4kVTUJUF_F0JCj84i-ow,4178
 regscale/integrations/commercial/wizv2/finding_processor.py,sha256=Rn2dYob4cz24_Ks-UTwNQe-KQ6hmFWS6GB8PQXV0ZPE,11361
@@ -329,7 +329,7 @@ regscale/models/integration_models/azure_alerts.py,sha256=2etrpvcxa7jVQrc98bJlVG
 regscale/models/integration_models/base64.py,sha256=sxV6O5qY1_TstJENX5jBPsSdQwmA83-NNhgJFunXiZE,570
 regscale/models/integration_models/burp.py,sha256=FBEBkH3U0Q8vq71FFoWnvgLRF5Hkr9GYmQFmNNHFrVk,16932
 regscale/models/integration_models/burp_models.py,sha256=UytDTAcCaxyu-knFkm_mEUH6UmWK3OTXKSC9Sc6OjVs,3669
-regscale/models/integration_models/cisa_kev_data.json,sha256=ThBF9xf2lRbdlWY1ucsdJU7u6ysXCBqMG80S9MMds2s,1268609
+regscale/models/integration_models/cisa_kev_data.json,sha256=2ex0O40ni9KX_RDUEoGIOEeKPELclHfeVehSDv6DJkM,1273678
 regscale/models/integration_models/defender_data.py,sha256=jsAcjKxiGmumGerj7xSWkFd6r__YpuKDnYX5o7xHDiE,2844
 regscale/models/integration_models/defenderimport.py,sha256=Ze4kgwns-IYPyO7sBjEzW8PXWlxwU-DAo2fIyRcTC3k,6242
 regscale/models/integration_models/drf.py,sha256=Aq7AdLa_CH97NrnR-CxaFI22JjVN9uCxVN7Z-BBUaNU,18896
@@ -360,7 +360,7 @@ regscale/models/integration_models/flat_file_importer/__init__.py,sha256=HYO7mrq
 regscale/models/integration_models/sbom/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 regscale/models/integration_models/sbom/cyclone_dx.py,sha256=0pFR0BWBrF5c8_cC_8mj2MXvNOMHOdHbBYXvTVfFAh8,4058
 regscale/models/integration_models/synqly_models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-regscale/models/integration_models/synqly_models/capabilities.json,sha256=Wcf7dY1zjVfdK5YRRq2lDK_bCPBI9k5TdKKczk4M2Qg,405530
+regscale/models/integration_models/synqly_models/capabilities.json,sha256=wNGPBgqYl1jfBoJzRNcXFoSRtQwK8I2cHUmHov8UYgc,408808
 regscale/models/integration_models/synqly_models/connector_types.py,sha256=8nxptkTexpskySnmL0obNAff_iu_fx6tJ7i1-4hJvao,461
 regscale/models/integration_models/synqly_models/filter_parser.py,sha256=8rdnHH7gW1A_uWRTexZXzCH-HzRVy5nlvFgtc7ztcsQ,12160
 regscale/models/integration_models/synqly_models/ocsf_mapper.py,sha256=ftObPhGg9CamnwRZ5z6qi8pW2Gu4JYy8apEo33o7q00,16960
@@ -493,7 +493,7 @@ regscale/utils/string.py,sha256=pY4eMcXRruCfGRaWKPN5NTWj6bXiKytxmU6XSxK4y6k,4198
 regscale/utils/synqly_utils.py,sha256=DjkdAxdUJBOU5YfAJ7pFFNBPSP37Gs9NQS10xduaYmA,6324
 regscale/utils/version.py,sha256=bOH1vlvOEAt1f-fzzFxtZrg3VcnqL5SA4ud1TjZVQGU,4265
 regscale/utils/threading/__init__.py,sha256=AuPiT3EijkUL4BSxtEQw3TrSvFW6oIfQzxtdy5sBCkk,296
-regscale/utils/threading/threadhandler.py,sha256=5lp9US6bKwh372UqrTvvHWIIodCcPIUmpcfGeCwG3VM,5061
+regscale/utils/threading/threadhandler.py,sha256=6K3f74A8nMCRhtz3XlCA0_Wqhx1L1pizYwdwoQLI4I8,5255
 regscale/utils/threading/threadsafe_counter.py,sha256=Dwt-jt4nN-ApMgpxONM7RFll6FU4hMCeZCW54rjDPjs,1037
 regscale/utils/threading/threadsafe_dict.py,sha256=XCVN5DoJsgZj-0tkDYAZb2QTLTZF9wS_ve-XKFJiywg,7656
 regscale/utils/threading/threadsafe_list.py,sha256=-ltEj-MQH0dN0z_5aalptwJw2_EAU6nn4RX32BDrhyI,2274
@@ -555,9 +555,9 @@ tests/regscale/models/test_regscale_model.py,sha256=ZsrEZkC4EtdIsoQuayn1xv2gEGcV
 tests/regscale/models/test_report.py,sha256=IqUq7C__a1_q_mLaz0PE9Lq6fHggBsB14-AzEYNBxLw,4666
 tests/regscale/models/test_tenable_integrations.py,sha256=y1qaW77H094VSGHjZdlvF66UCt-nPEib9Mv3cdwbM94,32435
 tests/regscale/models/test_user_model.py,sha256=e9olv28qBApgnvK6hFHOgXjUC-pkaV8aGDirEIWASL4,4427
-regscale_cli-6.25.0.1.dist-info/LICENSE,sha256=ytNhYQ9Rmhj_m-EX2pPq9Ld6tH5wrqqDYg-fCf46WDU,1076
-regscale_cli-6.25.0.1.dist-info/METADATA,sha256=o0IM1S-QKAG4kmMmru9qfqBykFJjs1F3HYJDRRKMG_c,35027
-regscale_cli-6.25.0.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-regscale_cli-6.25.0.1.dist-info/entry_points.txt,sha256=cLOaIP1eRv1yZ2u7BvpE3aB4x3kDrDwkpeisKOu33z8,269
-regscale_cli-6.25.0.1.dist-info/top_level.txt,sha256=Uv8VUCAdxRm70bgrD4YNEJUmDhBThad_1aaEFGwRByc,15
-regscale_cli-6.25.0.1.dist-info/RECORD,,
+regscale_cli-6.25.1.0.dist-info/LICENSE,sha256=ytNhYQ9Rmhj_m-EX2pPq9Ld6tH5wrqqDYg-fCf46WDU,1076
+regscale_cli-6.25.1.0.dist-info/METADATA,sha256=VTNaM1w0tCru4UYRJFjuEBku-SEdjzRF0ITFAW2WMkE,35027
+regscale_cli-6.25.1.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+regscale_cli-6.25.1.0.dist-info/entry_points.txt,sha256=cLOaIP1eRv1yZ2u7BvpE3aB4x3kDrDwkpeisKOu33z8,269
+regscale_cli-6.25.1.0.dist-info/top_level.txt,sha256=Uv8VUCAdxRm70bgrD4YNEJUmDhBThad_1aaEFGwRByc,15
+regscale_cli-6.25.1.0.dist-info/RECORD,,