cli2 4.3.3__tar.gz → 5.0.0__tar.gz

{cli2-4.3.3/cli2.egg-info → cli2-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cli2
-Version: 4.3.3
+Version: 5.0.0
 Summary: image:: https://yourlabs.io/oss/cli2/badges/master/pipeline.svg
 Home-page: https://yourlabs.io/oss/cli2
 Author: James Pic
@@ -13,9 +13,10 @@ Requires-Dist: docstring_parser
 Requires-Dist: pyyaml
 Requires-Dist: pygments
 Requires-Dist: structlog
-Provides-Extra: client
-Requires-Dist: httpx; extra == "client"
-Requires-Dist: truststore; extra == "client"
+Provides-Extra: httpx
+Requires-Dist: chttpx; extra == "httpx"
+Provides-Extra: ansible
+Requires-Dist: cansible; extra == "ansible"
 Provides-Extra: test
 Requires-Dist: freezegun; extra == "test"
 Requires-Dist: pytest; extra == "test"

{cli2-4.3.3 → cli2-5.0.0}/cli2/__init__.py

@@ -9,35 +9,10 @@ from .cli import (
     Group,
     EntryPoint,
 )
-from .asyncio import async_resolve, async_run
+from .asyncio import async_resolve, async_run, Queue
 from .colors import colors as c
 
 from .configuration import Configuration, cfg
-try:
-    from .client import (
-        ClientError,
-        ResponseError,
-        TokenGetError,
-        RefusedResponseError,
-        RetriesExceededError,
-        FieldError,
-        FieldValueError,
-        FieldExternalizeError,
-        Client,
-        ClientCommand,
-        DateTimeField,
-        Field,
-        Handler,
-        JSONStringField,
-        Model,
-        ModelCommand,
-        Paginator,
-        Related,
-    )
-except ImportError:
-    raise
-    # httpx not installed
-    pass
 from .display import diff, diff_data, render, print, highlight, yaml_highlight
 from .lock import Lock
 from .log import configure, log
@@ -55,3 +30,16 @@ def which(cmd):
     path = Path(os.getenv('HOME')) / '.local/bin' / cmd
     if path.exists():
         return str(path)
+
+
+def mutable(obj):
+    types = (
+        int,
+        float,
+        str,
+        tuple,
+        frozenset,
+        bool,
+        bytes,
+    )
+    return not isinstance(obj, types)

cli2-5.0.0/cli2/asyncio.py

@@ -0,0 +1,130 @@
+import asyncio
+import inspect
+from . import display
+from .log import log
+
+
+def async_iter(obj):
+    """ Check if an object is an async iterable. """
+    return inspect.isasyncgen(obj) or hasattr(obj, '__aiter__')
+
+
+async def async_resolve(result, output=False):
+    """
+    Recursively resolve awaitables and async iterables.
+
+    :param result: The awaitable or async iterable to resolve
+    :param output: If True, print results as they are resolved. If False,
+                   collect results.
+
+    :return: The resolved value(s). If output is True, returns None. If output
+             is False, returns a list of resolved values from async iterables.
+    """
+    while inspect.iscoroutine(result):
+        result = await result
+    if async_iter(result):
+        results = []
+        async for _ in result:
+            if output:
+                if (
+                    not inspect.iscoroutine(_)
+                    and not inspect.isasyncgen(_)
+                ):
+                    display.print(_)
+                else:
+                    await async_resolve(_, output=output)
+            else:
+                results.append(await async_resolve(_))
+        return None if output else results
+    return result
+
+
+def async_run(coroutine):
+    """Run an async coroutine in the current event loop or create a new one.
+
+    If an event loop is already running, creates a task in that loop.
+    If no event loop is running, creates a new one and runs the coroutine.
+
+    :param coroutine: The coroutine to run (return value of an async function)
+    :return: The result of the coroutine execution
+    """
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coroutine)
+    else:
+        return loop.create_task(coroutine)
+
+
+class Queue(asyncio.Queue):
+    """
+    An async queue with worker pool for concurrent task processing.
+
+    Extends asyncio.Queue to manage a pool of worker tasks that process items
+    from the queue concurrently.
+
+    .. code-block:: python
+
+        # will run 2 at the time
+        queue = cli2.Queue(num_workers=2)
+        # call like asyncio.run
+        await queue.run(foo(), bar(), other())
+
+    .. py:attribute:: num_workers
+
+        Number of concurrent workers (default: 12)
+
+    .. py:attribute:: results
+
+        List of results from completed tasks, order of results not garanteed
+        due to concurrency.
+    """
+
+    def __init__(self, *args, num_workers=12, **kwargs):
+        """Initialize the queue with worker pool.
+
+        :param num_workers: Number of concurrent workers (default: 12)
+        :paarm *args: Positional arguments for asyncio.Queue
+        :param **kwargs: Keyword arguments for asyncio.Queue
+        """
+        self.num_workers = num_workers or 12
+        self.results = []
+        super().__init__(*args, **kwargs)
+
+    async def run(self, *tasks):
+        """
+        Run tasks through the worker pool.
+
+        :param tasks: Coroutines
+        """
+        self.results = []
+
+        for task in tasks:
+            await self.put(task)
+
+        workers = [
+            asyncio.create_task(self.worker())
+            for i in range(self.num_workers)
+        ]
+
+        await self.join()
+        for worker in workers:
+            worker.cancel()
+
+    async def worker(self):
+        """Worker task that processes items from the queue.
+
+        Continuously gets tasks from the queue, executes them, and stores
+        results.
+        Handles exceptions by logging them.
+        """
+        while True:
+            task = await self.get()
+            try:
+                result = await task
+            except:  # noqa
+                log.exception()
+            else:
+                self.results.append(result)
+            finally:
+                self.task_done()

{cli2-4.3.3 → cli2-5.0.0}/cli2/display.py

@@ -7,16 +7,11 @@ Generic pretty display utils.
     force it with :envvar:`FORCE_COLOR`, ie. gitlab-ci etc
 """
 import difflib
+import json
 import os
 import sys
 import yaml
 
-try:
-    import jsonlight as json
-except ImportError:
-    import json
-
-
 _print = print
 
 

{cli2-4.3.3 → cli2-5.0.0}/cli2/mask.py

@@ -109,8 +109,10 @@ class Mask:
                     data[key] = self._mask(value)
         elif isinstance(data, list):
             return [self._mask(item) for item in data]
+        elif isinstance(data, set):
+            return {self._mask(item) for item in data}
         elif isinstance(data, str):
-            for value in self.values:
+            for value in sorted(self.values, key=len, reverse=True):
                 data = data.replace(str(value), '***MASKED***')
         return data
 

{cli2-4.3.3 → cli2-5.0.0/cli2.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cli2
-Version: 4.3.3
+Version: 5.0.0
 Summary: image:: https://yourlabs.io/oss/cli2/badges/master/pipeline.svg
 Home-page: https://yourlabs.io/oss/cli2
 Author: James Pic
@@ -13,9 +13,10 @@ Requires-Dist: docstring_parser
 Requires-Dist: pyyaml
 Requires-Dist: pygments
 Requires-Dist: structlog
-Provides-Extra: client
-Requires-Dist: httpx; extra == "client"
-Requires-Dist: truststore; extra == "client"
+Provides-Extra: httpx
+Requires-Dist: chttpx; extra == "httpx"
+Provides-Extra: ansible
+Requires-Dist: cansible; extra == "ansible"
 Provides-Extra: test
 Requires-Dist: freezegun; extra == "test"
 Requires-Dist: pytest; extra == "test"

{cli2-4.3.3 → cli2-5.0.0}/cli2.egg-info/SOURCES.txt

@@ -6,7 +6,6 @@ cli2/__init__.py
 cli2/asyncio.py
 cli2/cli.py
 cli2/cli2.py
-cli2/client.py
 cli2/colors.py
 cli2/configuration.py
 cli2/decorators.py
@@ -24,12 +23,7 @@ cli2.egg-info/dependency_links.txt
 cli2.egg-info/entry_points.txt
 cli2.egg-info/requires.txt
 cli2.egg-info/top_level.txt
-cli2/ansible/__init__.py
-cli2/ansible/action.py
-cli2/ansible/playbook.py
-cli2/ansible/variables.py
 cli2/examples/__init__.py
-cli2/examples/client.py
 cli2/examples/conf.py
 cli2/examples/example.py
 cli2/examples/example_obj.py

{cli2-4.3.3 → cli2-5.0.0}/cli2.egg-info/requires.txt

@@ -3,9 +3,11 @@ pyyaml
 pygments
 structlog
 
-[client]
-httpx
-truststore
+[ansible]
+cansible
+
+[httpx]
+chttpx
 
 [test]
 freezegun

{cli2-4.3.3 → cli2-5.0.0}/setup.py

@@ -44,7 +44,7 @@ from setuptools import setup
 
 setup(
     name='cli2',
-    version='4.3.3',
+    version='5.0.0',
     setup_requires='setupmeta',
     packages=['cli2'],
     install_requires=[
@@ -54,10 +54,8 @@ setup(
         'structlog',
     ],
     extras_require=dict(
-        client=[
-            'httpx',
-            'truststore',
-        ],
+        httpx=['chttpx'],
+        ansible=['cansible'],
         test=[
             'freezegun',
             'pytest',

{cli2-4.3.3 → cli2-5.0.0}/tests/test_ansible.py

@@ -1,24 +1,23 @@
 import cli2
+import chttpx
 import httpx
 import mock
 import pytest
 import textwrap
 import yaml
 
-from cli2 import ansible
-
-
-class ActionModule(ansible.ActionBase):
-    mask_keys = ['a']
-
-    async def run_async(self):
-        self.result['x'] = dict(a='a', b='b', c='c', d='foo a rrr')
+import cansible
 
 
 @pytest.mark.asyncio
 async def test_mask(monkeypatch):
-    printer = mock.Mock()
-    monkeypatch.setattr(ActionModule, 'print', printer)
+    class ActionModule(cansible.ActionBase):
+        masked_keys = ['a']
+        print = mock.Mock()
+
+        async def run_async(self):
+            self.result['x'] = dict(a='a', b='b', c='c', d='foo a rrr')
+
     module = await ActionModule.run_test_async(facts=dict(mask_keys=['b']))
     # result is untouched
     assert module.result == {'x':
@@ -26,15 +25,27 @@ async def test_mask(monkeypatch):
     }
     # output has proper masking
     expected = "\x1b[94mx\x1b[39;49;00m:\x1b[37m\x1b[39;49;00m\n\x1b[37m    \x1b[39;49;00m\x1b[94ma\x1b[39;49;00m:\x1b[37m \x1b[39;49;00m\x1b[33m'\x1b[39;49;00m\x1b[33m***MASKED***\x1b[39;49;00m\x1b[33m'\x1b[39;49;00m\x1b[37m\x1b[39;49;00m\n\x1b[37m    \x1b[39;49;00m\x1b[94mb\x1b[39;49;00m:\x1b[37m \x1b[39;49;00m\x1b[33m'\x1b[39;49;00m\x1b[33m***MASKED***\x1b[39;49;00m\x1b[33m'\x1b[39;49;00m\x1b[37m\x1b[39;49;00m\n\x1b[37m    \x1b[39;49;00m\x1b[94mc\x1b[39;49;00m:\x1b[37m \x1b[39;49;00mc\x1b[37m\x1b[39;49;00m\n\x1b[37m    \x1b[39;49;00m\x1b[94md\x1b[39;49;00m:\x1b[37m \x1b[39;49;00mfoo ***MASKED*** rrr\x1b[37m\x1b[39;49;00m\n"  # noqa
-    printer.assert_called_once_with(expected)
+    module.print.assert_called_once_with(expected, mask=False)
+
+
+def test_subprocess_remote(playbook):
+    playbook.task_add(
+        'yourlabs.test.password_get',
+        no_log=True,
+    )
+    result = playbook()
+    expected = "foo:***MASKED***:bar\n\nansible_facts:\n\n    mask_values:\n\n    - \'***MASKED***\'\n\nsecret: \'***MASKED***\'\n\nstdout: foo:***MASKED***:bar"  # noqa
+    assert expected in result['stdout']
+
+
 
 
 @pytest.mark.asyncio
 async def test_response_error(httpx_mock):
-    class Client(cli2.Client):
+    class Client(chttpx.Client):
         mask_keys = ['secret']
 
-    class Action(ansible.ActionBase):
+    class Action(cansible.ActionBase):
         async def client_factory(self):
             return Client(base_url='http://foo')
 
@@ -59,9 +70,9 @@ async def test_response_error(httpx_mock):
 
 @pytest.mark.asyncio
 async def test_option():
-    class Action(ansible.ActionBase):
-        fact = ansible.Option(fact='fact', default='default fact')
-        arg = ansible.Option(arg='arg', fact='arg_fact')
+    class Action(cansible.ActionBase):
+        fact = cansible.Option(fact='fact', default='default fact')
+        arg = cansible.Option(arg='arg', fact='arg_fact')
 
         async def run_async(self):
             self.result['arg'] = self.arg
@@ -90,8 +101,8 @@ async def test_option():
         error="Missing arg `arg` or fact `arg_fact`",
     )
 
-    class Action(ansible.ActionBase):
-        name = ansible.Option('name')
+    class Action(cansible.ActionBase):
+        name = cansible.Option('name')
 
         async def run_async(self):
             self.result['name'] = self.name
@@ -108,8 +119,8 @@ async def test_diff(monkeypatch):
     _print = mock.Mock()
     monkeypatch.setattr(cli2.display, '_print', _print)
 
-    class Action(ansible.ActionBase):
-        name = ansible.Option('name', 'object_name', 'Test object')
+    class Action(cansible.ActionBase):
+        name = cansible.Option('name', 'object_name', 'Test object')
 
         async def run_async(self):
             self.before_set(dict(foo=1))
@@ -119,6 +130,41 @@ async def test_diff(monkeypatch):
     _print.assert_called_once_with(expected)
 
 
+@pytest.mark.asyncio
+async def test_fact_set():
+    class Action(cansible.ActionBase):
+        foo = cansible.Option(fact='foo')
+
+        async def run_async(self):
+            self.foo = 'bar'
+            assert self.foo == 'bar'
+
+    action = await Action.run_test_async(facts=dict(foo='bar'))
+    assert 'ansible_facts' not in action.result
+
+    action = await Action.run_test_async()
+    assert action.result['ansible_facts'] == dict(foo='bar')
+
+
+@pytest.mark.asyncio
+async def test_fact_set_mutable():
+    class Action(cansible.ActionBase):
+        async def run_async(self):
+            assert self.mask_values is self.mask.values
+            self.mask_values.add('foo')
+
+    action = await Action.run_test_async()
+    assert 'mask_values' in action.facts_values
+    assert action.result['ansible_facts'] == dict(mask_values=['foo'])
+
+    action = await Action.run_test_async(facts=dict(mask_values=['foo']))
+    assert 'ansible_facts' not in action.result
+
+    action = await Action.run_test_async(facts=dict(mask_values=['bar']))
+    result = action.result['ansible_facts']['mask_values']
+    assert sorted(result) == ['bar', 'foo']
+
+
 def test_playbook_render(playbook):
     assert playbook.name == 'test_playbook_render'
     playbook.task_add(
@@ -165,3 +211,12 @@ def test_playbook_exec(playbook):
     result = playbook()
     assert result['changed'] == 0
     assert result['ok'] == 2
+
+
+def test_masking_ansible_story(playbook):
+    with open('tests/test_ansible_masking.yml', 'r') as f:
+        data = yaml.safe_load(f.read())
+    playbook.tasks += data[0]['tasks']
+    result = playbook()
+    expected = "hello ***MASKED*** ***MASKED*** bye\n\nansible_facts:\n\n    mask_values:\n\n    - \'***MASKED***\'\n\n    - \'***MASKED***\'\n\ncmd: echo hello ***MASKED*** ***MASKED*** bye\n\nstdout: hello ***MASKED*** ***MASKED*** bye"  # noqa
+    assert expected in result['stdout']

{cli2-4.3.3 → cli2-5.0.0}/tests/test_ansible_variables.py

@@ -1,10 +1,10 @@
-from cli2 import ansible
+import cansible
 import pytest
 import os
 
 
 def test_story():
-    variables = ansible.Variables(
+    variables = cansible.Variables(
         root_path=os.path.dirname(__file__),
         pass_path=os.path.dirname(__file__) + '/vault_pass',
     )
@@ -13,27 +13,34 @@ def test_story():
     assert variables['variables_vault.yml'] == dict(bar='foo')
 
 
-def test_exceptions():
-    variables = ansible.Variables()
-    with pytest.raises(Exception) as exc:
+def test_unresolvable_path_error():
+    variables = cansible.Variables()
+    with pytest.raises(cansible.UnresolvablePathError) as exc:
         variables['variables.yml']
     assert exc.value.args == (
         'variables.yml must be absolute if root_path not set',
     )
 
-    with pytest.raises(Exception) as exc:
-        variables['/variables_vault.yml']
-    assert exc.value.args == ('/variables_vault.yml does not exist',)
 
-    variables = ansible.Variables(root_path=os.path.dirname(__file__))
-    with pytest.raises(Exception) as exc:
+def test_path_not_found_error():
+    variables = cansible.Variables()
+    with pytest.raises(cansible.PathNotFoundError) as exc:
+        variables['/nonexistent.yml']
+    assert exc.value.args == ('/nonexistent.yml does not exist',)
+
+
+def test_vault_password_required_error():
+    variables = cansible.Variables(root_path=os.path.dirname(__file__))
+    with pytest.raises(cansible.VaultPasswordFileRequiredError) as exc:
         variables['variables_vault.yml']
     assert exc.value.args == ('Vault password required in pass_path',)
 
-    variables = ansible.Variables(
+
+def test_vault_password_file_not_found_error():
+    variables = cansible.Variables(
         root_path=os.path.dirname(__file__),
         pass_path='/does/not/exist',
     )
-    with pytest.raises(Exception) as exc:
+    with pytest.raises(cansible.VaultPasswordFileNotFoundError) as exc:
         variables['variables_vault.yml']
     assert exc.value.args == ('/does/not/exist does not exist',)

cli2-5.0.0/tests/test_asyncio.py

@@ -0,0 +1,68 @@
+import asyncio
+import cli2
+import pytest
+from unittest import mock
+
+
+def test_async_run_noloop():
+    async def task():
+        await asyncio.sleep(.01)
+
+    cli2.async_run(task())
+
+
+@pytest.mark.asyncio
+async def test_async_run_loop():
+    async def task():
+        await asyncio.sleep(.01)
+
+    cli2.async_run(task())
+
+
+@pytest.mark.asyncio
+async def test_queue_basic():
+    async def task(i):
+        return i
+
+    queue = cli2.Queue()
+    await queue.run(task(1), task(2), task(3))
+
+    assert sorted(queue.results) == [1, 2, 3]
+
+
+@pytest.mark.asyncio
+async def test_queue_concurrency():
+    start = asyncio.get_event_loop().time()
+
+    async def worker(i):
+        await asyncio.sleep(0.1)
+        return i
+
+    queue = cli2.Queue(num_workers=3)
+    await queue.run(worker(1), worker(2), worker(3))
+
+    duration = asyncio.get_event_loop().time() - start
+    assert duration < 0.15, 'Should run in parallel'
+    assert sorted(queue.results) == [1, 2, 3]
+
+
+@pytest.mark.asyncio
+async def test_queue_error_handling(monkeypatch):
+    import cli2.asyncio
+    logger = mock.Mock()
+    monkeypatch.setattr(cli2.asyncio, 'log', logger)
+
+    async def worker(i):
+        if i == 2:
+            raise ValueError('test error')
+        await asyncio.sleep(0.01)
+        return i
+
+    queue = cli2.Queue()
+    await queue.run(worker(1), worker(2), worker(3))
+
+    # Error should not stop other tasks
+    assert queue.results == [1, 3]
+
+    # Exceptions should be printed though, not swallowed
+    logger.exception.assert_called_once()

{cli2-4.3.3 → cli2-5.0.0}/tests/test_cli.py

@@ -47,7 +47,7 @@ def test_cli(name, command):
 
 
 def test_hide():
-    autotest('tests/test_hide.txt', 'cli2-example-client object find --help')
+    autotest('tests/test_hide.txt', 'chttpx-example object find --help')
 
 
 def test_load_module():