toil 5.11.0__tar.gz → 5.12.0__tar.gz

{toil-5.11.0/src/toil.egg-info → toil-5.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: toil
-Version: 5.11.0
+Version: 5.12.0
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community

toil-5.12.0/requirements-aws.txt

@@ -0,0 +1,4 @@
+boto>=2.48.0, <3
+boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3.post2, <2
+mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced
+moto>=4.1.11, <5

{toil-5.11.0 → toil-5.12.0}/requirements-cwl.txt

@@ -1,4 +1,4 @@
-cwltool==3.1.20230425144158
+cwltool==3.1.20230601100705
 schema-salad>=8.4.20230128170514,<9
 galaxy-tool-util
 ruamel.yaml>=0.15,<=0.17.21

{toil-5.11.0 → toil-5.12.0}/requirements-dev.txt

@@ -1,4 +1,4 @@
-mock>=4.0.3,<5
+mock>=4.0.3,<6
 pytest>=6.2.1,<8
 pytest-cov>=2.12.1,<5
 pytest-timeout>=1.4.2,<3

{toil-5.11.0 → toil-5.12.0}/requirements-google.txt

@@ -1,2 +1,3 @@
 apache-libcloud>=2.2.1,<3
 google-cloud-storage>=2,<=2.8.0
+google-auth>=2.18.1,<3

{toil-5.11.0 → toil-5.12.0}/requirements-wdl.txt

@@ -1,2 +1,2 @@
-miniwdl==1.9.1
+miniwdl==1.10.0
 wdlparse==0.1.0

{toil-5.11.0 → toil-5.12.0}/setup.cfg

@@ -13,7 +13,8 @@ markers =
 	docker_cuda
 	encryption
 	fetchable_appliance
-	google
+	google-project
+	google-storage
 	gridengine
 	htcondor
 	integrative

{toil-5.11.0 → toil-5.12.0}/src/toil/batchSystems/singleMachine.py

@@ -601,7 +601,7 @@ class SingleMachineBatchSystem(BatchSystemSupport):
         if job_accelerators:
             # Try and find some accelerators to use.
             # Start with all the accelerators that are free right now
-            accelerator_set : ResourceSet = self.resource_sources[3]
+            accelerator_set : ResourceSet = self.resource_sources[-1]
             snapshot = accelerator_set.get_free_snapshot()
             # And build a plan of the ones we want
             accelerators_needed, problem = self._identify_sufficient_accelerators(job_accelerators, snapshot)

{toil-5.11.0 → toil-5.12.0}/src/toil/common.py

@@ -528,7 +528,7 @@ def addOptions(parser: ArgumentParser, config: Optional[Config] = None, jobstore
     core_options.add_argument("--workDir", dest="workDir", default=None,
                               help="Absolute path to directory where temporary files generated during the Toil "
                                    "run should be placed. Standard output and error from batch system jobs "
-                                   "(unless --noStdOutErr) will be placed in this directory. A cache directory "
+                                   "(unless --noStdOutErr is set) will be placed in this directory. A cache directory "
                                    "may be placed in this directory. Temp files and folders will be placed in a "
                                    "directory toil-<workflowID> within workDir. The workflowID is generated by "
                                    "Toil and will be reported in the workflow logs. Default is determined by the "
@@ -1266,27 +1266,55 @@ class Toil(ContextManager["Toil"]):
     def import_file(self,
                     src_uri: str,
                     shared_file_name: str,
-                    symlink: bool = True) -> None: ...
+                    symlink: bool = True,
+                    check_existence: bool = True) -> None: ...
 
     @overload
     def import_file(self,
                     src_uri: str,
                     shared_file_name: None = None,
-                    symlink: bool = True) -> FileID: ...
+                    symlink: bool = True,
+                    check_existence: bool = True) -> FileID: ...
 
     def import_file(self,
                     src_uri: str,
                     shared_file_name: Optional[str] = None,
-                    symlink: bool = True) -> Optional[FileID]:
+                    symlink: bool = True,
+                    check_existence: bool = True) -> Optional[FileID]:
         """
         Import the file at the given URL into the job store.
 
+        By default, returns None if the file does not exist.
+
+        :param check_existence: If true, raise FileNotFoundError if the file
+               does not exist. If false, return None when the file does not
+               exist.
+
         See :func:`toil.jobStores.abstractJobStore.AbstractJobStore.importFile` for a
         full description
         """
         self._assertContextManagerUsed()
-        src_uri = self.normalize_uri(src_uri, check_existence=True)
-        return self._jobStore.import_file(src_uri, shared_file_name=shared_file_name, symlink=symlink)
+        full_uri = self.normalize_uri(src_uri, check_existence=check_existence)
+        try:
+            imported = self._jobStore.import_file(full_uri, shared_file_name=shared_file_name, symlink=symlink)
+        except FileNotFoundError:
+            # TODO: I thought we refactored the different job store import
+            # methods to not raise and instead return None, but that looks to
+            # not currently be the case.
+            if check_existence:
+                raise
+            else:
+                # So translate the raise-based API if needed.
+                # TODO: If check_existence is false but a shared file name is
+                # specified, we have no way to report the lack of file
+                # existence, since we also return None on success!
+                return None
+        if imported is None and shared_file_name is None and check_existence:
+            # We need to protect the caller from missing files.
+            # We think a file was missing, and we got None becasuse of it.
+            # We didn't get None instead because of usign a shared file name.
+            raise FileNotFoundError(f'Could not find file {src_uri}')
+        return imported
 
     @deprecated(new_function_name='export_file')
     def exportFile(self, jobStoreFileID: FileID, dstUrl: str) -> None:
@@ -1308,7 +1336,7 @@ class Toil(ContextManager["Toil"]):
         """
         Given a URI, if it has no scheme, prepend "file:".
 
-        :param check_existence: If set, raise an error if a URI points to
+        :param check_existence: If set, raise FileNotFoundError if a URI points to
                a local file that does not exist.
         """
         if urlparse(uri).scheme == 'file':

{toil-5.11.0 → toil-5.12.0}/src/toil/cwl/__init__.py

@@ -15,15 +15,17 @@ import logging
 from functools import lru_cache
 
 from pkg_resources import DistributionNotFound, get_distribution
+
 try:
     # Setuptools 66+ will raise this if any package on the system has a version that isn't PEP440.
     # See https://github.com/pypa/setuptools/issues/3772
-    from setuptools.extern.packaging.version import InvalidVersion # type: ignore
+    from setuptools.extern.packaging.version import InvalidVersion  # type: ignore
 except ImportError:
     # It's not clear that this exception is really part fo the public API, so fake it.
-    class InvalidVersion(Exception): # type: ignore
+    class InvalidVersion(Exception):  # type: ignore
         pass
 
+
 from toil.version import cwltool_version
 
 logger = logging.getLogger(__name__)
@@ -47,8 +49,10 @@ def check_cwltool_version() -> None:
     except DistributionNotFound:
         logger.debug("cwltool is not installed.")
     except InvalidVersion as e:
-        logger.warning(f"Could not determine the installed version of cwltool because a package "
-                       f"with an unacceptable version is installed: {e}")
+        logger.warning(
+            f"Could not determine the installed version of cwltool because a package "
+            f"with an unacceptable version is installed: {e}"
+        )
 
 
 check_cwltool_version()

{toil-5.11.0 → toil-5.12.0}/src/toil/cwl/cwltoil.py

@@ -60,10 +60,10 @@ import cwl_utils.expression
 import cwltool.builder
 import cwltool.command_line_tool
 import cwltool.context
+import cwltool.cwlprov
 import cwltool.job
 import cwltool.load_tool
 import cwltool.main
-import cwltool.provenance
 import cwltool.resolver
 import schema_salad.ref_resolver
 from cwltool.loghandler import _logger as cwllogger
@@ -85,9 +85,9 @@ from cwltool.software_requirements import (
 )
 from cwltool.stdfsaccess import StdFsAccess, abspath
 from cwltool.utils import (
-    DirectoryType,
     CWLObjectType,
     CWLOutputType,
+    DirectoryType,
     adjustDirObjs,
     aslist,
     downloadHttpFile,
@@ -110,6 +110,7 @@ from toil.cwl.utils import (
     download_structure,
     visit_cwl_class_and_reduce,
 )
+from toil.exceptions import FailedJobsException
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import AcceleratorRequirement, Job, Promise, Promised, unwrap
@@ -119,8 +120,6 @@ from toil.jobStores.utils import JobStoreUnavailableException, generate_locator
 from toil.lib.threading import ExceptionalThread
 from toil.statsAndLogging import DEFAULT_LOGLEVEL
 from toil.version import baseVersion
-from toil.exceptions import FailedJobsException
-
 
 logger = logging.getLogger(__name__)
 
@@ -2015,7 +2014,8 @@ def toilStageFiles(
                             f.close()
                             # Import it and pack up the file ID so we can turn around and export it.
                             file_id_or_contents = (
-                                "toilfile:" + toil.import_file(f.name, symlink=False).pack()
+                                "toilfile:"
+                                + toil.import_file(f.name, symlink=False).pack()
                             )
 
                     if file_id_or_contents.startswith("toildir:"):
@@ -3625,7 +3625,7 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
     loading_context = cwltool.main.setup_loadingContext(None, runtime_context, options)
 
     if options.provenance:
-        research_obj = cwltool.provenance.ResearchObject(
+        research_obj = cwltool.cwlprov.ro.ResearchObject(
             temp_prefix_ro=options.tmp_outdir_prefix,
             orcid=options.orcid,
             full_name=options.cwl_full_name,
@@ -3708,8 +3708,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
             document_loader = loading_context.loader
 
             if options.provenance and runtime_context.research_obj:
-                runtime_context.research_obj.packed_workflow(
-                    cwltool.main.print_pack(loading_context, uri)
+                cwltool.cwlprov.writablebagfile.packed_workflow(
+                    runtime_context.research_obj,
+                    cwltool.main.print_pack(loading_context, uri),
                 )
 
             try:
@@ -3879,7 +3880,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
         toilStageFiles(toil, outobj, outdir, destBucket=options.destBucket)
 
         if runtime_context.research_obj is not None:
-            runtime_context.research_obj.create_job(outobj, True)
+            cwltool.cwlprov.writablebagfile.create_job(
+                runtime_context.research_obj, outobj, True
+            )
 
             def remove_at_id(doc: Any) -> None:
                 if isinstance(doc, MutableMapping):
@@ -3906,7 +3909,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
                 workflowobj, document_loader, uri
             )
             runtime_context.research_obj.generate_snapshot(prov_dependencies)
-            runtime_context.research_obj.close(options.provenance)
+            cwltool.cwlprov.writablebagfile.close_ro(
+                runtime_context.research_obj, options.provenance
+            )
 
         if not options.destBucket and options.compute_checksum:
             visit_class(

{toil-5.11.0 → toil-5.12.0}/src/toil/fileStores/abstractFileStore.py

@@ -346,7 +346,14 @@ class AbstractFileStore(ABC):
                 # For each access record
                 if len(item) == 2:
                     # If it has a name, dump wit the name
-                    logger.warning('Downloaded file \'%s\' to path \'%s\'', *item)
+                    file_id, dest_path = item
+                    if os.path.exists(dest_path):
+                        if os.path.islink(dest_path):
+                            logger.warning('Symlinked file \'%s\' to path \'%s\'', file_id, dest_path)
+                        else:
+                            logger.warning('Downloaded file \'%s\' to path \'%s\'', file_id, dest_path)
+                    else:
+                        logger.warning('Downloaded file \'%s\' to path \'%s\' (gone!)', file_id, dest_path)
                 else:
                     # Otherwise dump without the name
                     logger.warning('Streamed file \'%s\'', *item)

{toil-5.11.0 → toil-5.12.0}/src/toil/jobStores/abstractJobStore.py

@@ -43,11 +43,10 @@ else:
     from typing_extensions import Literal
 
 from urllib.parse import ParseResult, urlparse
+from urllib.error import HTTPError
 from urllib.request import urlopen
 from uuid import uuid4
 
-from requests.exceptions import HTTPError
-
 from toil.common import Config, getNodeID, safeUnpickleFromStream
 from toil.fileStores import FileID
 from toil.job import (CheckpointJobDescription,
@@ -420,6 +419,8 @@ class AbstractJobStore(ABC):
             - 'gs'
                 e.g. gs://bucket/file
 
+        Raises FileNotFoundError if the file does not exist.
+
         :param str src_uri: URL that points to a file or object in the storage mechanism of a
                 supported URL scheme e.g. a blob in an AWS s3 bucket. It must be a file, not a
                 directory or prefix.
@@ -453,6 +454,8 @@ class AbstractJobStore(ABC):
         asks the other job store class for a stream and writes that stream as either a regular or
         a shared file.
 
+        Raises FileNotFoundError if the file does not exist.
+
         :param AbstractJobStore otherCls: The concrete subclass of AbstractJobStore that supports
                reading from the given URL and getting the file size from the URL.
 
@@ -587,6 +590,8 @@ class AbstractJobStore(ABC):
         """
         Read the given URL and write its content into the given writable stream.
 
+        Raises FileNotFoundError if the URL doesn't exist.
+
         :return: The size of the file in bytes and whether the executable permission bit is set
         :rtype: Tuple[int, bool]
         """
@@ -618,8 +623,12 @@ class AbstractJobStore(ABC):
         Reads the contents of the object at the specified location and writes it to the given
         writable stream.
 
+        Raises FileNotFoundError if the URL doesn't exist.
+
         Refer to :func:`~AbstractJobStore.importFile` documentation for currently supported URL schemes.
 
+        Raises FileNotFoundError if the thing at the URL is not found.
+
         :param ParseResult url: URL that points to a file or object in the storage
                mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
 
@@ -635,7 +644,7 @@ class AbstractJobStore(ABC):
     def _write_to_url(cls, readable: Union[IO[bytes], IO[str]], url: ParseResult, executable: bool = False) -> None:
         """
         Reads the contents of the given readable stream and writes it to the object at the
-        specified location.
+        specified location. Raises FileNotFoundError if the URL doesn't exist..
 
         Refer to AbstractJobStore.importFile documentation for currently supported URL schemes.
 
@@ -1707,20 +1716,28 @@ class JobStoreSupport(AbstractJobStore, metaclass=ABCMeta):
         # We can only retry on errors that happen as responses to the request.
         # If we start getting file data, and the connection drops, we fail.
         # So we don't have to worry about writing the start of the file twice.
-        with closing(urlopen(url.geturl())) as readable:
-            # Make something to count the bytes we get
-            # We need to put the actual count in a container so our
-            # nested function can modify it without creating its own
-            # local with the same name.
-            size = [0]
-            def count(l: int) -> None:
-                size[0] += l
-            counter = WriteWatchingStream(writable)
-            counter.onWrite(count)
-
-            # Do the download
-            shutil.copyfileobj(readable, counter)
-            return size[0], False
+        try:
+            with closing(urlopen(url.geturl())) as readable:
+                # Make something to count the bytes we get
+                # We need to put the actual count in a container so our
+                # nested function can modify it without creating its own
+                # local with the same name.
+                size = [0]
+                def count(l: int) -> None:
+                    size[0] += l
+                counter = WriteWatchingStream(writable)
+                counter.onWrite(count)
+
+                # Do the download
+                shutil.copyfileobj(readable, counter)
+                return size[0], False
+        except HTTPError as e:
+            if e.code == 404:
+                # Translate into a FileNotFoundError for detecting
+                # un-importable files
+                raise FileNotFoundError(str(url)) from e
+            else:
+                raise
 
     @classmethod
     def _get_is_directory(cls, url: ParseResult) -> bool:

{toil-5.11.0 → toil-5.12.0}/src/toil/jobStores/googleJobStore.py

@@ -27,6 +27,7 @@ from google.api_core.exceptions import (GoogleAPICallError,
                                         InternalServerError,
                                         ServiceUnavailable)
 from google.cloud import exceptions, storage
+from google.auth.exceptions import DefaultCredentialsError
 
 from toil.jobStores.abstractJobStore import (AbstractJobStore,
                                              JobStoreExistsException,
@@ -112,25 +113,54 @@ class GoogleJobStore(AbstractJobStore):
         self.readStatsBaseID = self.statsReadPrefix+self.statsBaseID
 
         self.sseKey = None
+        self.storageClient = self.create_client()
 
-        # Determine if we have an override environment variable for our credentials.
-        # We don't pull out the filename; we just see if a name is there.
-        self.credentialsFromEnvironment = bool(os.getenv('GOOGLE_APPLICATION_CREDENTIALS', False))
 
-        if self.credentialsFromEnvironment and not os.path.exists(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')):
+    @classmethod
+    def create_client(cls) -> storage.Client:
+        """
+        Produce a client for Google Sotrage with the highest level of access we can get.
+
+        Fall back to anonymous access if no project is available, unlike the
+        Google Storage module's behavior.
+
+        Warn if GOOGLE_APPLICATION_CREDENTIALS is set but not actually present.
+        """
+
+        # Determine if we have an override environment variable for our credentials.
+        # We get the path to check existence, but Google Storage works out what
+        # to use later by looking at the environment again.
+        credentials_path: Optional[str] = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
+        if credentials_path is not None and not os.path.exists(credentials_path):
             # If the file is missing, complain.
             # This variable holds a file name and not any sensitive data itself.
             log.warning("File '%s' from GOOGLE_APPLICATION_CREDENTIALS is unavailable! "
                         "We may not be able to authenticate!",
-                        os.getenv('GOOGLE_APPLICATION_CREDENTIALS'))
-
-        if not self.credentialsFromEnvironment and os.path.exists(self.nodeServiceAccountJson):
-            # load credentials from a particular file on GCE nodes if an override path is not set
-            self.storageClient = storage.Client.from_service_account_json(self.nodeServiceAccountJson)
-        else:
-            # Either a filename is specified, or our fallback file isn't there.
+                        credentials_path)
+
+        if credentials_path is None and os.path.exists(cls.nodeServiceAccountJson):
+            try:
+                # load credentials from a particular file on GCE nodes if an override path is not set
+                return storage.Client.from_service_account_json(cls.nodeServiceAccountJson)
+            except OSError:
+                # Probably we don't have permission to use the file.
+                log.warning("File '%s' exists but didn't work to authenticate!",
+                            cls.nodeServiceAccountJson)
+                pass
+
+        # Either a filename is specified, or our fallback file isn't there.
+        try:
             # See if Google can work out how to authenticate.
-            self.storageClient = storage.Client()
+            return storage.Client()
+        except (DefaultCredentialsError, EnvironmentError):
+            # Depending on which Google codepath or module version (???)
+            # realizes we have no credentials, we can get an EnvironemntError,
+            # or the new DefaultCredentialsError we are supposedly specced to
+            # get.
+
+            # Google can't find credentials, fall back to being anonymous.
+            # This is likely to happen all the time so don't warn.
+            return storage.Client.create_anonymous_client()
 
 
     @google_retry
@@ -244,10 +274,11 @@ class GoogleJobStore(AbstractJobStore):
 
         env = {}
 
-        if self.credentialsFromEnvironment:
+        credentials_path: Optional[str] = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
+        if credentials_path is not None:
             # Send along the environment variable that points to the credentials file.
             # It must be available in the same place on all nodes.
-            env['GOOGLE_APPLICATION_CREDENTIALS'] = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+            env['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_path
 
         return env
 
@@ -351,8 +382,8 @@ class GoogleJobStore(AbstractJobStore):
         if fileName.startswith('/'):
             fileName = fileName[1:]
 
-        storageClient = storage.Client()
-        bucket = storageClient.get_bucket(bucketName)
+        storageClient = cls.create_client()
+        bucket = storageClient.bucket(bucket_name=bucketName)
         blob = bucket.blob(compat_bytes(fileName))
 
         if exists:

{toil-5.11.0 → toil-5.12.0}/src/toil/lib/accelerators.py

@@ -14,8 +14,9 @@
 
 """Accelerator (i.e. GPU) utilities for Toil"""
 
+import os
 import subprocess
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, Optional, Set, Union
 from xml.dom import minidom
 
 from toil.job import AcceleratorRequirement
@@ -37,6 +38,37 @@ def have_working_nvidia_smi() -> bool:
         return False
     return True
 
+@memoize
+def get_host_accelerator_numbers() -> List[int]:
+    """
+    Work out what accelerator is what.
+
+    For each accelerator visible to us, returns the host-side (for example,
+    outside-of-Slurm-job) number for that accelerator. It is often the same as
+    the apparent number.
+
+    Can be used with Docker's --gpus='"device=#,#,#"' option to forward the
+    right GPUs as seen from a Docker daemon.
+    """
+
+    for number_list_var in ['SLURM_STEP_GPUS', 'SLURM_JOB_GPUS', 'CUDA_VISIBLE_DEVICES', 'NVIDIA_VISIBLE_DEVICES']:
+        # Any of these can have a list of GPU numbers, but the CUDA/NVIDIA ones
+        # also support a system of GPU GUIDs that we don't support.
+        # TODO: If Slurm confinement is set we ignore any attempt to further
+        # limit us with the app-level variables. Does that make sense? Writing
+        # code to translate through would be hard and probably not actually
+        # useful.
+        if number_list_var in os.environ:
+            device_string = os.environ[number_list_var]
+            # Parse all the numbers we have
+            device_numbers = [int(part) for part in device_string.split(',') if part.isnumeric()]
+            if len(device_numbers) > 0:
+                # We found some numbers, so use those
+                return device_numbers
+
+    # If we don't see a set of limits we understand, say we have all nvidia GPUs
+    return list(range(count_nvidia_gpus()))
+
 @memoize
 def have_working_nvidia_docker_runtime() -> bool:
     """
@@ -45,7 +77,7 @@ def have_working_nvidia_docker_runtime() -> bool:
     try:
         # The runtime injects nvidia-smi; it doesn't seem to have to be in the image we use here
         subprocess.check_call(['docker', 'run', '--rm', '--runtime', 'nvidia', '--gpus', 'all', 'ubuntu:20.04', 'nvidia-smi'])
-    except subprocess.CalledProcessError:
+    except (FileNotFoundError, subprocess.CalledProcessError):
         return False
     return True
 
@@ -83,7 +115,7 @@ def get_individual_local_accelerators() -> List[AcceleratorRequirement]:
     # For now we only know abput nvidia GPUs
     return [{'kind': 'gpu', 'brand': 'nvidia', 'api': 'cuda', 'count': 1} for _ in range(count_nvidia_gpus())]
 
-def get_restrictive_environment_for_local_accelerators(accelerator_numbers : Set[int]) -> Dict[str, str]:
+def get_restrictive_environment_for_local_accelerators(accelerator_numbers : Union[Set[int], List[int]]) -> Dict[str, str]:
     """
     Get environment variables which can be applied to a process to restrict it
     to using only the given accelerator numbers.

{toil-5.11.0 → toil-5.12.0}/src/toil/lib/aws/iam.py

@@ -3,7 +3,7 @@ import json
 import logging
 from collections import defaultdict
 from functools import lru_cache
-from typing import Any, Dict, List, Optional, Set, cast
+from typing import Any, Dict, List, Optional, Set, cast, Union, Sequence
 
 import boto3
 from mypy_boto3_iam import IAMClient
@@ -80,8 +80,6 @@ def add_to_action_collection(a: AllowedActionCollection, b: AllowedActionCollect
     return to_return
 
 
-
-
 def policy_permissions_allow(given_permissions: AllowedActionCollection, required_permissions: List[str] = []) -> bool:
     """
     Check whether given set of actions are a subset of another given set of actions, returns true if they are
@@ -189,7 +187,23 @@ def allowed_actions_roles(iam: IAMClient, policy_names: List[str], role_name: st
 
     return allowed_actions
 
-def allowed_actions_users(iam: IAMClient, policy_names: List[str], user_name: str) -> AllowedActionCollection:
+
+def collect_policy_actions(policy_documents: Sequence[Union[str, Dict[str, Any]]]) -> AllowedActionCollection:
+    """
+    Collect all of the actions allowed by the given policy documents into one AllowedActionCollection.
+    """
+    allowed_actions: AllowedActionCollection = init_action_collection()
+    for policy_str in policy_documents:
+        # sometimes a string is returned from the api, so convert to a dictionary
+        if isinstance(policy_str, dict):
+            policy_dict = policy_str
+        else:
+            policy_dict = json.loads(policy_str)
+        allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_dict))
+    return allowed_actions
+
+
+def allowed_actions_user(iam: IAMClient, policy_names: List[str], user_name: str) -> AllowedActionCollection:
     """
     Gets all allowed actions for a user given by user_name, returns a dictionary, keyed by resource,
     with a list of permissions allowed for each given resource.
@@ -198,17 +212,34 @@ def allowed_actions_users(iam: IAMClient, policy_names: List[str], user_name: st
     :param policy_names: Name of policy document associated with a user
     :param user_name: Name of user to get associated policies
     """
-    allowed_actions: AllowedActionCollection = init_action_collection()
-
-    for policy_name in policy_names:
-        user_policy = iam.get_user_policy(
+    user_policies = [
+        iam.get_user_policy(
             UserName=user_name,
             PolicyName=policy_name
-        )
-        policy_document = json.loads(user_policy["PolicyDocument"])
-        allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_document))
+        )["PolicyDocument"]
+        for policy_name in policy_names
+    ]
+    return collect_policy_actions(user_policies)
+
+
+def allowed_actions_group(iam: IAMClient, policy_names: List[str], group_name: str) -> AllowedActionCollection:
+    """
+    Gets all allowed actions for a group given by group_name, returns a dictionary, keyed by resource,
+    with a list of permissions allowed for each given resource.
+
+    :param iam: IAM client to use
+    :param policy_names: Name of policy document associated with a user
+    :param group_name: Name of group to get associated policies
+    """
+    group_policies = [
+        iam.get_group_policy(
+            GroupName=group_name,
+            PolicyName=policy_name
+        )["PolicyDocument"]
+        for policy_name in policy_names
+    ]
+    return collect_policy_actions(group_policies)
 
-    return allowed_actions
 
 def get_policy_permissions(region: str) -> AllowedActionCollection:
     """
@@ -229,9 +260,19 @@ def get_policy_permissions(region: str) -> AllowedActionCollection:
         attached_policies = iam.list_attached_user_policies(UserName=user['User']['UserName'])
         user_attached_policies = allowed_actions_attached(iam, attached_policies['AttachedPolicies'])
         allowed_actions = add_to_action_collection(allowed_actions, user_attached_policies)
-        user_inline_policies = allowed_actions_users(iam, list_policies['PolicyNames'], user['User']['UserName'])
+        user_inline_policies = allowed_actions_user(iam, list_policies['PolicyNames'], user['User']['UserName'])
         allowed_actions = add_to_action_collection(allowed_actions, user_inline_policies)
 
+        # grab group policies associated with the user
+        groups = iam.list_groups_for_user(UserName=user['User']['UserName'])
+        for group in groups["Groups"]:
+            list_policies = iam.list_group_policies(GroupName=group['GroupName'])
+            attached_policies = iam.list_attached_group_policies(GroupName=group['GroupName'])
+            group_attached_policies = allowed_actions_attached(iam, attached_policies['AttachedPolicies'])
+            allowed_actions = add_to_action_collection(allowed_actions, group_attached_policies)
+            group_inline_policies = allowed_actions_group(iam, list_policies['PolicyNames'], group['GroupName'])
+            allowed_actions = add_to_action_collection(allowed_actions, group_inline_policies)
+
     except:
         # If not successful, we check the role associated with an instance profile
         # and grab the role's associated permissions