ciocore 8.0.0b27__py2.py3-none-any.whl → 8.0.0b29__py2.py3-none-any.whl

ciocore/downloader/download_runner_base.py

@@ -37,7 +37,7 @@ class DownloadRunnerBase(object):
         """
         Run the downloader.
 
-        Optionally wrap the ownloader in a reporter to report task statuses back to the server.
+        Optionally wrap the downloader in a reporter to report task statuses back to the server.
         """
         if self.disable_reporting:
             self.downloader.run()

{ciocore-8.0.0b27.dist-info → ciocore-8.0.0b29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 8.0.0b27
+Version: 8.0.0b29
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -51,63 +51,17 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
-## Unreleased: 
-* 8.0.0-beta.26
-  * Fix a bug where the perpetual downloader ignored Ctrl+C interrupts.
-  * Change destination flag to output_path so that later we can introduce real daemon mode.
-  * Report to server now reports tasks as downloaded even if they were skipped due to pre-existing.
-  * Report to server now respects the filter and reports tasks as downloaded if the desired subset of files were downloaded.
-  * Perpetual downloader prints an ascii spinner to the console.
-
-* 8.0.0-beta.24
-  * adds the ability to disable reporting with a cli flag and functionality in the download runner class.
-  * Update logger name and try to guard against multiple loggers
-  * Add file filtering functionality to BaseDownloader and its subclasses
-  * Ensure log to stdout so that it's greppable
-
-* 8.0.0-beta.20
-  * Colored logging and better error detection when request for url fails.
-  
-* 8.0.0-beta.19
-  * Refactor and improvements to threading and logging.
-  
-* 8.0.0-beta.18
-  * Adds robustness to handling of Ctrl+C, errors, threads, and logging.
-
-* 8.0.0-beta.17
-  * Ignores redundant tasks in the download queue
-
-* 8.0.0-beta.15
-  * remove sequence checker and unnecessary dependencies
-
-* 8.0.0-beta.14
-  * fixed permissions error bug with file downloads
-
-* 8.0.0-beta.13
-  * fixed permissions error bug with file downloads
-
-* 8.0.0-beta.5 
-  * adds kill_tasks kill_jobs get_jobs get_log
-  * remove old docs stuff
-
-* 8.0.0-beta.4 
-  * updates in changelog for dev
-  * dev updates in changelog
-  * minor
-  * refactor datacache to use tables
-  * some python 3 updates
-  * fixed bug where read_creds hung
-  * requests+chardet as charset_normalizer breaks m23
-  * Adds DataCache class and tests
-  * adds data cache instead of fixtures
-  * cull bad 3dsmax entries
-  * start data_cache
-  * tidy based on comparison with ref repo
-  * some formatting and fstrings
-  * minor uploader additions
-  * adds build_docs back in to cicd
-  * skulk 3 compatibility
-  * wip testing page downloader
+## Version: 8.0.0-beta.29 -- 30 Mar 2024
+
+Major Updates in Version 8.0.0
+
+
+* The `conductor` commandline interface (CLI) has undergone a complete overhaul to improve its functionality and performance. It is now installed as a Python console script, which simplifies the installation process on all platforms and ensures that the tool is always accessible. In addition, this standardized installation allows for better integration into our Companion application and potentially customers' own proprietary tools.
+* The commandline downloader has been rewritten to improve its performance and reliability. In previous versions, the downloader would fetch all download URLs from the server before starting any downloads. This approach was inefficient and would lead to performance issues when handling jobs with large numbers of tasks. The new downloader fetches tasks in small batches, which allows the download to start almost immediately, regardless of the number of tasks. It provides several new commandline flags, such as filtering based on filenames, and better logging, giving users more control and visibility over the download process. In addition, the new downloader has been written as an event based system that can be seamlessly integrated into other pipeline tools and GUIs. Pipeline TDs need only to register handlers to any of the events that the downloader emits.
+* The `conductor` commandline now supports a new subcommand, `docs`, which opens API and commandline documentation in a local web browser. The documentation is generated using `mkdocs` and is included in the package. This command is useful as it ensures the **API** documentation is relevant to the version of the tool being used.
+* The `conductor` commandline now supports a new subcommand, `packages`, which displays all available software packages on Conductor's cloud. The command can output the document as text, markdown, or nicely formatted HTML. The data is pulled from the Conductor server and is always up-to-date.
+* The Python API now provides four new endpoints: `kill_tasks`, `kill_jobs`, `get_jobs`, and `get_log`.
+* The Python API now provides access to entries in the Dashboard's Extra Environment Variables section. This feature allows administrators to set environment variables that are passed to all jobs run on the cloud. It's especially useful for setting up access to license servers or other shared resources.
 
 
 ## Version:7.0.2 -- 10 Nov 2023

{ciocore-8.0.0b27.dist-info → ciocore-8.0.0b29.dist-info}/RECORD

@@ -1,13 +1,13 @@
-ciocore/VERSION,sha256=c7X8SzkXuZYVK0o_s91vtvExswrXsAUEXRyKxnNLI8E,13
+ciocore/VERSION,sha256=XlqO4qN_smGJF38Bz0RcIlXfqSCxhHSod3Bi6Rj2Z8o,13
 ciocore/__init__.py,sha256=aTP7LeeosQA8BZE67gDV4jgfTK5zxmwZRjiTRu_ZWj0,646
-ciocore/api_client.py,sha256=BTDGXT-59-48SLz3BzP8OtGEhBYDiiEvyytsRUxaK9I,23555
-ciocore/cli.py,sha256=6DaZHUBWYkRjsFft5tLrEnp2CrfX5l1i1kAJvW6Z1iI,15400
+ciocore/api_client.py,sha256=TyHXGmK4uTmKV93O_IpAvttZhbexIIZTGP-IVn_WyWM,24599
+ciocore/cli.py,sha256=BXgULpqElTHwX6nUcSFVVh99MCBSG2FjuOzRKhIa6Hk,15399
 ciocore/client_db.py,sha256=tTz3bl2xeDPPcYSDS3g3QgV_xYihJMx0Kj6OeN2klK0,12978
 ciocore/common.py,sha256=FnggAL-IGW8VQB6kLsrKLnxWFgLpwGBhOtiZSHz130M,14725
 ciocore/compat.py,sha256=5uEXPSog_jxsDMaHBswAKEtfyXT25VgU6WNGIhz9PHU,256
 ciocore/conductor_submit.py,sha256=ONE0LsA5hGavTJIOXXYx8qzl8_vBPADwhd6Ytq_0E0c,9382
 ciocore/config.py,sha256=rCL7kaFn1tYgSglN8q9Wx6SwMpoXTq0BMQGwPRVwVIg,8973
-ciocore/data.py,sha256=pO_hQGI7dwHwmfkTUjVA3WOWGn4mYbsDUG5w7hlktZ4,6791
+ciocore/data.py,sha256=Ji0qUk8nJXBNakoHSqBiVx8O58SbZXyt273SHlEDn3U,7027
 ciocore/dev_inst_tagger.py,sha256=bPfoFbE7IPCpDbtbVHxW7IlMwvspdnJhPuNq-u4fJN4,3497
 ciocore/exceptions.py,sha256=4Oq-WX-qiN6kPUdBCHvvd6mtSQ0nCkDbJxWt2CNtpv8,1504
 ciocore/file_utils.py,sha256=bAlL31B4YkRgX-yT8kF8UXBFktQlsE1PvxbKqTeAeOU,17174
@@ -25,13 +25,13 @@ ciocore/auth/server.py,sha256=8btX9-EokUl6q55V8muDmEV2tvvbTBD0BHeWFbwkzUc,3892
 ciocore/docsite/404.html,sha256=DQCrQsaKh8M0zlCFZmjFHUtqy7_gx_i2Rpc3OoDdDI4,17207
 ciocore/docsite/index.html,sha256=p_Dq_6_8cwZZPDI5zjahfCWUkfbSj14NJtoQFAZv0WI,20945
 ciocore/docsite/logo.png,sha256=gArgFFWdw8w985-0TkuGIgU_pW9sziEMZdqytXb5WLo,2825
-ciocore/docsite/objects.inv,sha256=UhlZWOqRV8tA4CzS8HKmk322X-so5ZmIe55155t_7Ls,746
+ciocore/docsite/objects.inv,sha256=s2FKStLlVIQbfG7U2-nw-7rz4unvd1W0u00YtLBxAKo,758
 ciocore/docsite/sitemap.xml,sha256=M_V85zl0y2adRvzJAnoCxlZH_Hl7TLnIb1A-6l_xGmI,109
-ciocore/docsite/sitemap.xml.gz,sha256=uAtX2Tv_tKPdyEqEhzZCgoUBWL0e42nZNlvEWVk6qEY,127
-ciocore/docsite/apidoc/api_client/index.html,sha256=DYVIV4xZRhTEiDJoIRn-76YRjGia_ce6LDss7G2stsE,164621
+ciocore/docsite/sitemap.xml.gz,sha256=w5DLJrlr2lrhIadg85ABMEdEM0uhOIlOUoYF7J_4zjQ,127
+ciocore/docsite/apidoc/api_client/index.html,sha256=mHDq-zdWuilDKUluzmQNdOsImRoGEnnK5Ul2ApajgdU,170218
 ciocore/docsite/apidoc/apidoc/index.html,sha256=NQn8wjapxa7O2IQhsbE7Y-VMfMFL6Tby-L7qIF6K3m4,26171
 ciocore/docsite/apidoc/config/index.html,sha256=1GWkyClM7LJlLBpx07BqCi3xDkU-SKVPjNUEeKuwbNs,72559
-ciocore/docsite/apidoc/data/index.html,sha256=c7FJVeXgqlaj5f_BTndVJfjADUkEJ7PyV3lLUfmiR8I,50095
+ciocore/docsite/apidoc/data/index.html,sha256=9DonjpASPYZGYRHjyaodkK3AxSlTHsvm1xdq2cqMxPI,50850
 ciocore/docsite/apidoc/hardware_set/index.html,sha256=DMlpzQCy2ypJ078tcFNrZAvG2sU7P5-ZwU4ZRCObtQg,123042
 ciocore/docsite/apidoc/package_environment/index.html,sha256=S5IE1EgnJaaLVsEpgPogwBvj3jxT6ycNp-1GAlJrPt4,65729
 ciocore/docsite/apidoc/package_tree/index.html,sha256=wKW5dsHBlcDyvCGakSNzYHX8z99F1Vhu2NyqAW6c1ko,109393
@@ -82,12 +82,12 @@ ciocore/docsite/cmdline/downloader/index.html,sha256=DAVErhWEPuEtkQr7y9D_wjylzM9
 ciocore/docsite/cmdline/packages/index.html,sha256=j1YeMgdwZ6FyJGJii05AdtUWKRjyu1fOr3_bKJsLfEU,20923
 ciocore/docsite/cmdline/uploader/index.html,sha256=y0728lIAYC4b9mfpjNVfwQqXoqJrsdG49UY5u7B1p94,25123
 ciocore/docsite/how-to-guides/index.html,sha256=OFrFBTb9tuMn60Jd4oeM10Mw-d-0pyqwoKoo9g1biBM,20100
-ciocore/docsite/search/search_index.json,sha256=t-RJBVqxWPIzkjKMY9dMfpuwzVnJIeHkrZ6eWHrmkLM,179918
+ciocore/docsite/search/search_index.json,sha256=8R5VcEveS35Oe-LACgZF4QjnvO6ds5HhAhyAlRMjOeA,181518
 ciocore/docsite/stylesheets/extra.css,sha256=qgfcao9TEBzf6ieAWN5rPyrea7YM_YgVg5qr4NPrxoU,354
 ciocore/docsite/stylesheets/tables.css,sha256=O2PEwlKC0RfvZCRV9UERqQRvhb6jcqO84PLew0bafF0,3279
 ciocore/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ciocore/downloader/base_downloader.py,sha256=9PVmP1qy-dop4MbVsmLFvf5W6LKQLUKHPDYE6AEt71g,25784
-ciocore/downloader/download_runner_base.py,sha256=8TCRfDLMt0jk00T6Zg-YDwDp5VS8J9OqYg_caYCNd70,1611
+ciocore/downloader/download_runner_base.py,sha256=cWTWKhFX5FLG84Wh8_s7xpk3n0nmQ4fDiD2rYV6a3u0,1612
 ciocore/downloader/job_downloader.py,sha256=HAhr95RyCsj3KTp5W58Znc4JmqHY-mrM5b8vo6kRIJ0,5697
 ciocore/downloader/legacy_downloader.py,sha256=ef9hCMUTTuxYHKzpEoK5bc3gqwAEIhewkC_KpbhLVHE,52087
 ciocore/downloader/log.py,sha256=WCRNx0LObx8JBO5MQucNHQGBXMgSBLGdH0VALN8dFDo,2105
@@ -100,11 +100,12 @@ ciocore/uploader/_uploader.py,sha256=AaCwRo1rGkdMA-XACx9k4Z73R97ndJUVMemUZNV-Ark
 ciocore/uploader/upload_stats/__init__.py,sha256=Lg1y4zq1i0cwc6Hh2K1TAQDYymLff49W-uIo1xjcvdI,5309
 ciocore/uploader/upload_stats/stats_formats.py,sha256=giNirtObU66VALWghPFSRhg3q_vw5MvESsnXhb_I3y8,2402
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/extra_env_fixtures.py,sha256=8qvU4d8SXGKzRVNR5whVqKCQOwOVMiFVfbKBAjxa2gE,1119
 tests/instance_type_fixtures.py,sha256=uIzQduqKQVgjllMuyXaYnRC-pwqk5lnTx3NY2M5Nujo,4320
 tests/package_fixtures.py,sha256=CsJnhB7oYzIxJH7b1tCOPyvnnVSCqEbSPhtCnsHL-nA,5070
 tests/project_fixtures.py,sha256=iBm_th_JtAw76vlNu7Jjhh9tLH4oOaNi-MgtPzCV7yQ,138
-tests/test_api_client.py,sha256=lDUkVi6Xq1ilkSepFLebgBAhgchumvfyZnTtei1fnrI,3642
-tests/test_base_downloader.py,sha256=oJg0a-lDh8xcQIw9ZnkTHvUyOEw0k2AW7UqAX4IyjpI,3695
+tests/test_api_client.py,sha256=tnYkIYvDILGJRatZM0nrtr83x8yBJstMP0Rt151U2gI,6757
+tests/test_base_downloader.py,sha256=SS7tWKv2ZZhpUDk4UCg1TkrNrpntjSewgzLl1mEubSE,3603
 tests/test_cli.py,sha256=yWWGLip8WgnDAB0vg7uhBU5eeEJieFLQ78Pn7IN-uWI,6088
 tests/test_common.py,sha256=tY_-SY-JmJX09UehFs9RIDqZ785AmhfTl6eVKJeIUFY,763
 tests/test_config.py,sha256=-_G682Ss3Zr1FmcMkjla1zAZprX2tQKpKc5_wD28rII,13340
@@ -120,8 +121,8 @@ tests/test_uploader.py,sha256=B1llTJt_fqR6e_V_Jxfw9z73QgkFlEPU87xLYGzt-TQ,2914
 tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-8.0.0b27.dist-info/METADATA,sha256=jFWGG_mM8rn3Ndauf6UPE3LyhUFwCpRYwIgJT5V1nMI,17427
-ciocore-8.0.0b27.dist-info/WHEEL,sha256=iYlv5fX357PQyRT2o6tw1bN-YcKFFHKqB_LwHO5wP-g,110
-ciocore-8.0.0b27.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
-ciocore-8.0.0b27.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-8.0.0b27.dist-info/RECORD,,
+ciocore-8.0.0b29.dist-info/METADATA,sha256=RhJwGlupsp9OkYWYYaHTlZXpn_yNBG0stXRyo8KZacA,17850
+ciocore-8.0.0b29.dist-info/WHEEL,sha256=iYlv5fX357PQyRT2o6tw1bN-YcKFFHKqB_LwHO5wP-g,110
+ciocore-8.0.0b29.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
+ciocore-8.0.0b29.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-8.0.0b29.dist-info/RECORD,,

tests/extra_env_fixtures.py

@@ -0,0 +1,57 @@
+
+
+EXTRA_ENV = [
+    {
+      "account_id": "6649535867387904",
+      "env": [],
+    },
+    {
+      "account_id": "5767615549800448",
+      "env": [
+        {
+          "merge_policy": "prepend",
+          "name": "test",
+          "value": "test"
+        }
+      ],
+    },
+    {
+      "account_id": "6649535867387904",
+      "env": [],
+    },
+    {
+      "account_id": "5669544198668288",
+      "env": [
+        {
+          "merge_policy": "append",
+          "name": "PATH",
+          "value": "/path/to/scripts"
+        },
+        {
+          "merge_policy": "exclusive",
+          "name": "RENDER_LOCATION",
+          "value": "cloud"
+        },
+        {
+          "merge_policy": "exclusive",
+          "name": "VARIABLE_USED_IN_SCRIPTS",
+          "value": "true"
+        },
+        {
+          "merge_policy": "exclusive",
+          "name": "testvar",
+          "value": "somevalue"
+        }
+      ],
+    },
+    {
+      "account_id": "6649535867387904",
+      "env": [
+        {
+          "merge_policy": "exclusive",
+          "name": "JMEYER",
+          "value": "JMEYER_ENV_VALUE"
+        }
+      ],
+    }
+  ]

tests/test_api_client.py

@@ -4,7 +4,7 @@
 """
 import sys
 import unittest
-
+import json
 try:
     from unittest import mock
 except ImportError:
@@ -12,6 +12,7 @@ except ImportError:
 
 from ciocore import api_client
 
+from ciocore.api_client import request_extra_environment
 
 class ApiClientTest(unittest.TestCase):
     @staticmethod
@@ -105,3 +106,61 @@ class TestRegisterClient(unittest.TestCase):
 
         self.assertTrue(user_agent.startswith(expected_user_agent))
 
+
+class TestRequestExtraEnvironment(unittest.TestCase):
+
+    def setUp(self):
+        self.api_response_data = {
+            "data": [
+                {"account_id": "123", "env": ["VAR1=value1", "VAR2=value2"]},
+                {"account_id": "456", "env": ["VAR3=value3", "VAR4=value4"]}
+            ]
+        }
+        self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.api_response_data))
+        self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
+
+    @mock.patch("ciocore.api_client.ApiClient")
+    @mock.patch("ciocore.api_client.read_conductor_credentials")
+    @mock.patch("ciocore.api_client.account_id_from_jwt")
+    def test_request_extra_environment_success(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
+        # Set up mocks for successful execution
+        mock_read_conductor_credentials.return_value = "valid_token"
+        mock_account_id_from_jwt.return_value = "123"
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
+
+        result = request_extra_environment()
+
+        self.assertEqual(result, ["VAR1=value1", "VAR2=value2"])
+        mock_ApiClient.assert_called_once()
+        mock_read_conductor_credentials.assert_called_once_with(True)
+        mock_account_id_from_jwt.assert_called_once_with("valid_token")
+
+    @mock.patch("ciocore.api_client.ApiClient")
+    def test_request_extra_environment_api_failure(self, mock_ApiClient):
+        # Set up mock for API failure
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_error.text, self.response_error.status_code)
+
+        # Assert exception raised when the API call fails
+        with self.assertRaises(Exception) as context:
+            request_extra_environment()
+
+        self.assertIn('Failed to get extra environment', str(context.exception))
+        mock_ApiClient.assert_called_once()
+
+    @mock.patch("ciocore.api_client.ApiClient")
+    @mock.patch("ciocore.api_client.read_conductor_credentials")
+    @mock.patch("ciocore.api_client.account_id_from_jwt")
+    def test_request_extra_environment_no_account_env(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
+        # Set up mocks to simulate valid token and account ID but no matching environment
+        mock_read_conductor_credentials.return_value = "valid_token"
+        mock_account_id_from_jwt.return_value = "invalid_id"  # This won't match any 'account_id' in response
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
+
+        with self.assertRaises(Exception) as context:
+            request_extra_environment()
+
+        self.assertEqual("Error: Could not get account environment!", str(context.exception))
+        mock_ApiClient.assert_called_once()

tests/test_base_downloader.py

@@ -9,11 +9,11 @@ from ciocore.downloader.base_downloader import (
 )
 import unittest
 from unittest import mock
-from ciocore.downloader.base_downloader import temp_file
+
 from ciocore import api_client
-import unittest
-from unittest.mock import Mock, patch
-import threading
+
+from unittest.mock import  patch
+
 from concurrent.futures import ThreadPoolExecutor