cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

cartography/client/core/tx.py

@@ -13,7 +13,11 @@ from cartography.models.core.nodes import CartographyNodeSchema
 from cartography.util import batch
 
 
-def read_list_of_values_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[Union[str, int]]:
+def read_list_of_values_tx(
+    tx: neo4j.Transaction,
+    query: str,
+    **kwargs,
+) -> List[Union[str, int]]:
     """
     Runs the given Neo4j query in the given transaction object and returns a list of either str or int. This is intended
     to be run only with queries that return a list of a single field.
@@ -39,7 +43,11 @@ def read_list_of_values_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[
     return values
 
 
-def read_single_value_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Optional[Union[str, int]]:
+def read_single_value_tx(
+    tx: neo4j.Transaction,
+    query: str,
+    **kwargs,
+) -> Optional[Union[str, int]]:
     """
     Runs the given Neo4j query in the given transaction object and returns a str, int, or None. This is intended to be
     run only with queries that return a single str, int, or None value.
@@ -70,7 +78,11 @@ def read_single_value_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Optiona
     return value
 
 
-def read_list_of_dicts_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[Dict[str, Any]]:
+def read_list_of_dicts_tx(
+    tx: neo4j.Transaction,
+    query: str,
+    **kwargs,
+) -> List[Dict[str, Any]]:
     """
     Runs the given Neo4j query in the given transaction object and returns the results as a list of dicts.
 
@@ -92,7 +104,11 @@ def read_list_of_dicts_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[D
     return values
 
 
-def read_list_of_tuples_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[Tuple[Any, ...]]:
+def read_list_of_tuples_tx(
+    tx: neo4j.Transaction,
+    query: str,
+    **kwargs,
+) -> List[Tuple[Any, ...]]:
     """
     Runs the given Neo4j query in the given transaction object and returns the results as a list of tuples.
 
@@ -154,9 +170,9 @@ def read_single_dict_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Any:
 
 
 def write_list_of_dicts_tx(
-        tx: neo4j.Transaction,
-        query: str,
-        **kwargs,
+    tx: neo4j.Transaction,
+    query: str,
+    **kwargs,
 ) -> None:
     """
     Writes a list of dicts to Neo4j.
@@ -192,10 +208,10 @@ def write_list_of_dicts_tx(
 
 
 def load_graph_data(
-        neo4j_session: neo4j.Session,
-        query: str,
-        dict_list: List[Dict[str, Any]],
-        **kwargs,
+    neo4j_session: neo4j.Session,
+    query: str,
+    dict_list: List[Dict[str, Any]],
+    **kwargs,
 ) -> None:
     """
     Writes data to the graph.
@@ -215,7 +231,10 @@ def load_graph_data(
         )
 
 
-def ensure_indexes(neo4j_session: neo4j.Session, node_schema: CartographyNodeSchema) -> None:
+def ensure_indexes(
+    neo4j_session: neo4j.Session,
+    node_schema: CartographyNodeSchema,
+) -> None:
     """
     Creates indexes if they don't exist for the given CartographyNodeSchema object, as well as for all of the
     relationships defined on its `other_relationships` and `sub_resource_relationship` fields. This operation is
@@ -229,16 +248,18 @@ def ensure_indexes(neo4j_session: neo4j.Session, node_schema: CartographyNodeSch
     queries = build_create_index_queries(node_schema)
 
     for query in queries:
-        if not query.startswith('CREATE INDEX IF NOT EXISTS'):
-            raise ValueError('Query provided to `ensure_indexes()` does not start with "CREATE INDEX IF NOT EXISTS".')
+        if not query.startswith("CREATE INDEX IF NOT EXISTS"):
+            raise ValueError(
+                'Query provided to `ensure_indexes()` does not start with "CREATE INDEX IF NOT EXISTS".',
+            )
         neo4j_session.run(query)
 
 
 def load(
-        neo4j_session: neo4j.Session,
-        node_schema: CartographyNodeSchema,
-        dict_list: List[Dict[str, Any]],
-        **kwargs,
+    neo4j_session: neo4j.Session,
+    node_schema: CartographyNodeSchema,
+    dict_list: List[Dict[str, Any]],
+    **kwargs,
 ) -> None:
     """
     Main entrypoint for intel modules to write data to the graph. Ensures that indexes exist for the datatypes loaded

cartography/config.py

@@ -26,6 +26,8 @@ class Config:
     :type aws_sync_all_profiles: bool
     :param aws_sync_all_profiles: If True, AWS sync will run for all non-default profiles in the AWS_CONFIG_FILE. If
         False (default), AWS sync will run using the default credentials only. Optional.
+    :type aws_regions: str
+    :param aws_regions: Comma-separated list of AWS regions to sync. Optional.
     :type aws_best_effort_mode: bool
     :param aws_best_effort_mode: If True, AWS sync will not raise any exceptions, just log. If False (default),
         exceptions will be raised.
@@ -133,6 +135,7 @@ class Config:
         selected_modules=None,
         update_tag=None,
         aws_sync_all_profiles=False,
+        aws_regions=None,
         aws_best_effort_mode=False,
         azure_sync_all_subscriptions=False,
         azure_sp_auth=None,
@@ -194,6 +197,7 @@ class Config:
         self.selected_modules = selected_modules
         self.update_tag = update_tag
         self.aws_sync_all_profiles = aws_sync_all_profiles
+        self.aws_regions = aws_regions
         self.aws_best_effort_mode = aws_best_effort_mode
         self.azure_sync_all_subscriptions = azure_sync_all_subscriptions
         self.azure_sp_auth = azure_sp_auth

cartography/driftdetect/__main__.py

@@ -2,6 +2,5 @@ import sys
 
 import cartography.driftdetect.cli
 
-
-if __name__ == '__main__':
+if __name__ == "__main__":
     sys.exit(cartography.driftdetect.cli.main())

cartography/driftdetect/add_shortcut.py

@@ -22,7 +22,13 @@ def run_add_shortcut(config):
         logger.error("Invalid Drift Detection Directory")
         return
     try:
-        add_shortcut(FileSystem, ShortcutSchema(), config.query_directory, config.shortcut, config.filename)
+        add_shortcut(
+            FileSystem,
+            ShortcutSchema(),
+            config.query_directory,
+            config.shortcut,
+            config.filename,
+        )
     except ValidationError as err:
         msg = "Could not load shortcut file from json file {} in query directory {}.".format(
             err.messages,
@@ -48,7 +54,9 @@ def add_shortcut(storage, shortcut_serializer, query_directory, alias, filename)
     :return:
     """
     if storage.has_file(os.path.join(query_directory, alias)):
-        logger.error(f"Shortcut {alias} is the name of another File in directory {query_directory}.")
+        logger.error(
+            f"Shortcut {alias} is the name of another File in directory {query_directory}.",
+        )
         return
     shortcut_path = os.path.join(query_directory, "shortcut.json")
     shortcut_data = storage.load(shortcut_path)

cartography/driftdetect/cli.py

@@ -8,7 +8,6 @@ from cartography.driftdetect.add_shortcut import run_add_shortcut
 from cartography.driftdetect.detect_deviations import run_drift_detection
 from cartography.driftdetect.get_states import run_get_states
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -26,138 +25,130 @@ class CLI:
         parser = argparse.ArgumentParser(
             prog=self.prog,
             description=(
-                'drift-detection takes database queries along with their expected states in the cartography-generated '
-                'graph database and reports the deviations.'
+                "drift-detection takes database queries along with their expected states in the cartography-generated "
+                "graph database and reports the deviations."
             ),
-            epilog='For more documentation please visit: '
-                   'https://cartography-cncf.github.io/cartography/usage/drift-detect.html',
+            epilog="For more documentation please visit: "
+            "https://cartography-cncf.github.io/cartography/usage/drift-detect.html",
         )
         parser.add_argument(
-            '-v',
-            '--verbose',
-            action='store_true',
-            help='Enable verbose logging for drift-detection.',
+            "-v",
+            "--verbose",
+            action="store_true",
+            help="Enable verbose logging for drift-detection.",
         )
         parser.add_argument(
-            '-q',
-            '--quiet',
-            action='store_true',
-            help='Restrict drift-detection logging to warnings and errors only.',
+            "-q",
+            "--quiet",
+            action="store_true",
+            help="Restrict drift-detection logging to warnings and errors only.",
         )
         subparsers = parser.add_subparsers(
-            dest='command',
-            help='To get the current drift state, use the command `cartography-detectdrift get-state --neo4j_uri <your '
-                 'neo4j_uri> --drift-detection-directory <your drift detection directory>` \n'
-                 'To get drift between two state files, use the command `cartography-detectdrift get-drift'
-                 '--query-directory <path to query directory> --start-state <beginning drift state file> --end-state '
-                 '<final drift state file>'
-                 'To add a shortcut between two state files, use the command `cartography-detectdrift add-shortcut'
-                 '--query-directory <path to query directory> --shortcut <shortcut name> --file <driftstate filename>',
+            dest="command",
+            help="To get the current drift state, use the command `cartography-detectdrift get-state --neo4j_uri <your "
+            "neo4j_uri> --drift-detection-directory <your drift detection directory>` \n"
+            "To get drift between two state files, use the command `cartography-detectdrift get-drift"
+            "--query-directory <path to query directory> --start-state <beginning drift state file> --end-state "
+            "<final drift state file>"
+            "To add a shortcut between two state files, use the command `cartography-detectdrift add-shortcut"
+            "--query-directory <path to query directory> --shortcut <shortcut name> --file <driftstate filename>",
         )
         parser_get_state = subparsers.add_parser(
-            name='get-state',
-            help='generates new drift state for each query with the current status of the neo4j database',
+            name="get-state",
+            help="generates new drift state for each query with the current status of the neo4j database",
         )
         parser_get_state.add_argument(
-            '--neo4j-uri',
+            "--neo4j-uri",
             type=str,
-            default='bolt://localhost:7687',
+            default="bolt://localhost:7687",
             help=(
-                'A valid Neo4j URI to sync against. See '
-                'https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the '
-                'structure of a Neo4j URI.'
+                "A valid Neo4j URI to sync against. See "
+                "https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the "
+                "structure of a Neo4j URI."
             ),
         )
         parser_get_state.add_argument(
-            '--neo4j-user',
+            "--neo4j-user",
             type=str,
             default=None,
-            help='A username with which to authenticate to Neo4j.',
+            help="A username with which to authenticate to Neo4j.",
         )
         parser_get_state.add_argument(
-            '--neo4j-password-env-var',
+            "--neo4j-password-env-var",
             type=str,
             default=None,
-            help='The name of an environment variable containing a password with which to authenticate to Neo4j.',
+            help="The name of an environment variable containing a password with which to authenticate to Neo4j.",
         )
         parser_get_state.add_argument(
-            '--neo4j-password-prompt',
-            action='store_true',
+            "--neo4j-password-prompt",
+            action="store_true",
             help=(
-                'Present an interactive prompt for a password with which to authenticate to Neo4j. This parameter '
-                'supersedes other methods of supplying a Neo4j password.'
+                "Present an interactive prompt for a password with which to authenticate to Neo4j. This parameter "
+                "supersedes other methods of supplying a Neo4j password."
             ),
         )
         parser_get_state.add_argument(
-            '--drift-detection-directory',
+            "--drift-detection-directory",
             type=str,
             default=None,
             help=(
-                'A path to a directory containing drift-states to build. Drift-detection will discover all JSON'
-                'files in the given directory (and its subdirectories) and construct detectors from'
-                'them. Drift-detection does not guarantee the order in which the detector jobs are executed.'
+                "A path to a directory containing drift-states to build. Drift-detection will discover all JSON"
+                "files in the given directory (and its subdirectories) and construct detectors from"
+                "them. Drift-detection does not guarantee the order in which the detector jobs are executed."
             ),
         )
         parser_get_drift = subparsers.add_parser(
-            name='get-drift',
+            name="get-drift",
             help=(
-                'gets drift between two drift states. Must be between the same detection directory'
+                "gets drift between two drift states. Must be between the same detection directory"
             ),
         )
         parser_get_drift.add_argument(
-            '--query-directory',
+            "--query-directory",
             type=str,
             default=None,
             help=(
-                'A path to a directory containing drift-states for a specific query. Drift-detection will read in the'
-                'report_info file and specified drift-states from the directory to report the differences between files'
+                "A path to a directory containing drift-states for a specific query. Drift-detection will read in the"
+                "report_info file and specified drift-states from the directory to report the differences between files"
             ),
         )
         parser_get_drift.add_argument(
-            '--start-state',
+            "--start-state",
             type=str,
             default=None,
             help=(
-                'The filename of the earlier state chronologically to be compared to.'
+                "The filename of the earlier state chronologically to be compared to."
             ),
         )
         parser_get_drift.add_argument(
-            '--end-state',
+            "--end-state",
             type=str,
             default=None,
-            help=(
-                'The filename of the later state chronologically to be compared to.'
-            ),
+            help=("The filename of the later state chronologically to be compared to."),
         )
         parser_add_shortcut = subparsers.add_parser(
-            name='add-shortcut',
-            help=(
-                'Adds a shortcut to a specific file in a query directory.'
-            ),
+            name="add-shortcut",
+            help=("Adds a shortcut to a specific file in a query directory."),
         )
         parser_add_shortcut.add_argument(
-            '--query-directory',
+            "--query-directory",
             type=str,
             default=None,
             help=(
-                'A path to a directory containing drift-states for a specific query.'
+                "A path to a directory containing drift-states for a specific query."
             ),
         )
         parser_add_shortcut.add_argument(
-            '--shortcut',
+            "--shortcut",
             type=str,
             default=None,
-            help=(
-                'The desired alias for the filename.'
-            ),
+            help=("The desired alias for the filename."),
         )
         parser_add_shortcut.add_argument(
-            '--filename',
+            "--filename",
             type=str,
             default=None,
-            help=(
-                'The desired name of the file to be replaced.'
-            ),
+            help=("The desired name of the file to be replaced."),
         )
         return parser
 
@@ -171,13 +162,13 @@ class CLI:
         # TODO support parameter lookup in environment variables if not present on command line
         config = self.parser.parse_args(argv)
         if config.verbose:
-            logging.getLogger('driftdetect').setLevel(logging.DEBUG)
+            logging.getLogger("driftdetect").setLevel(logging.DEBUG)
         elif config.quiet:
-            logging.getLogger('driftdetect').setLevel(logging.WARNING)
+            logging.getLogger("driftdetect").setLevel(logging.WARNING)
         else:
-            logging.getLogger('driftdetect').setLevel(logging.INFO)
+            logging.getLogger("driftdetect").setLevel(logging.INFO)
         logger.debug("Launching driftdetect with CLI configuration: %r", vars(config))
-        if config.command == 'get-state':
+        if config.command == "get-state":
             config = configure_get_state_neo4j(config)
         return config
 
@@ -190,11 +181,11 @@ class CLI:
         """
         config = self.configure(argv)
         try:
-            if config.command == 'get-state':
+            if config.command == "get-state":
                 run_get_states(config)
-            elif config.command == 'get-drift':
+            elif config.command == "get-drift":
                 run_drift_detection(config)
-            elif config.command == 'add-shortcut':
+            elif config.command == "add-shortcut":
                 run_add_shortcut(config)
             else:
                 msg = "No command detected. Try --help."
@@ -215,7 +206,10 @@ def configure_get_state_neo4j(config):
     if config.neo4j_user:
         config.neo4j_password = None
         if config.neo4j_password_prompt:
-            logger.info("Reading password for Neo4j user '%s' interactively.", config.neo4j_user)
+            logger.info(
+                "Reading password for Neo4j user '%s' interactively.",
+                config.neo4j_user,
+            )
             config.neo4j_password = getpass.getpass()
         elif config.neo4j_password_env_var:
             logger.debug(
@@ -225,7 +219,9 @@ def configure_get_state_neo4j(config):
             )
             config.neo4j_password = os.environ.get(config.neo4j_password_env_var)
         if not config.neo4j_password:
-            logger.warning("Neo4j username was provided but a password could not be found.")
+            logger.warning(
+                "Neo4j username was provided but a password could not be found.",
+            )
     else:
         config.neo4j_password = None
     return config
@@ -239,6 +235,6 @@ def main(argv=None):
     :return: The return code.
     """
     logging.basicConfig(level=logging.INFO)
-    logging.getLogger('neo4j').setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.WARNING)
     argv = argv if argv is not None else sys.argv[1:]
     return CLI(prog="cartography-detectdrift").main(argv)

cartography/driftdetect/detect_deviations.py

@@ -23,11 +23,14 @@ def run_drift_detection(config: GetDriftConfig) -> None:
             return
         state_serializer = StateSchema()
         shortcut_serializer = ShortcutSchema()
-        shortcut_data = FileSystem.load(os.path.join(config.query_directory, "shortcut.json"))
+        shortcut_data = FileSystem.load(
+            os.path.join(config.query_directory, "shortcut.json"),
+        )
         shortcut = shortcut_serializer.load(shortcut_data)
         start_state_data = FileSystem.load(
             os.path.join(
-                config.query_directory, shortcut.shortcuts.get(
+                config.query_directory,
+                shortcut.shortcuts.get(
                     config.start_state,
                     config.start_state,
                 ),
@@ -36,7 +39,8 @@ def run_drift_detection(config: GetDriftConfig) -> None:
         start_state = state_serializer.load(start_state_data)
         end_state_data = FileSystem.load(
             os.path.join(
-                config.query_directory, shortcut.shortcuts.get(
+                config.query_directory,
+                shortcut.shortcuts.get(
                     config.end_state,
                     config.end_state,
                 ),

cartography/driftdetect/get_states.py

@@ -74,13 +74,25 @@ def run_get_states(config: UpdateConfig) -> None:
         return
 
     with neo4j_driver.session() as session:
-        filename = '.'.join([str(i) for i in time.gmtime()] + ["json"])
+        filename = ".".join([str(i) for i in time.gmtime()] + ["json"])
         state_serializer = StateSchema()
         shortcut_serializer = ShortcutSchema()
         for query_directory in FileSystem.walk(config.drift_detection_directory):
             try:
-                get_query_state(session, query_directory, state_serializer, FileSystem, filename)
-                add_shortcut(FileSystem, shortcut_serializer, query_directory, 'most-recent', filename)
+                get_query_state(
+                    session,
+                    query_directory,
+                    state_serializer,
+                    FileSystem,
+                    filename,
+                )
+                add_shortcut(
+                    FileSystem,
+                    shortcut_serializer,
+                    query_directory,
+                    "most-recent",
+                    filename,
+                )
             except ValidationError as err:
                 msg = "Unable to create State for directory {}, with data \n{}".format(
                     query_directory,
@@ -97,11 +109,11 @@ def run_get_states(config: UpdateConfig) -> None:
 
 
 def get_query_state(
-        session: neo4j.Session,
-        query_directory: str,
-        state_serializer: StateSchema,
-        storage,
-        filename: str,
+    session: neo4j.Session,
+    query_directory: str,
+    state_serializer: StateSchema,
+    storage,
+    filename: str,
 ) -> State:
     """
     Gets the most recent state of a query.

cartography/driftdetect/model.py

@@ -19,11 +19,11 @@ class State:
     """
 
     def __init__(
-            self,
-            name: str,
-            validation_query: str,
-            properties: List[str],
-            results: List[List[str]],
+        self,
+        name: str,
+        validation_query: str,
+        properties: List[str],
+        results: List[List[str]],
     ):
 
         self.name: str = name

cartography/driftdetect/serializers.py

@@ -10,6 +10,7 @@ class StateSchema(Schema):
     """
     Schema to serialize and deserialize DriftStates from JSON.
     """
+
     name = fields.Str()
     validation_query = fields.Str()
     properties = fields.List(fields.Str())
@@ -18,10 +19,10 @@ class StateSchema(Schema):
     @post_load
     def make_state(self, data, **kwargs):
         return State(
-            data['name'],
-            data['validation_query'],
-            data['properties'],
-            data['results'],
+            data["name"],
+            data["validation_query"],
+            data["properties"],
+            data["results"],
         )
 
 
@@ -29,12 +30,13 @@ class ShortcutSchema(Schema):
     """
     Schema to serialize and deserialize Shortcuts from JSON.
     """
+
     name = fields.Str()
     shortcuts = fields.Dict(keys=fields.Str(), values=fields.Str())
 
     @post_load
     def make_misc(self, data, **kwargs):
         return Shortcut(
-            data['name'],
-            data['shortcuts'],
+            data["name"],
+            data["shortcuts"],
         )

cartography/driftdetect/storage.py

@@ -25,9 +25,9 @@ class FileSystem:
         :param file_path: Filepath to be written to.
         :return:
         """
-        with open(file_path, 'w') as json_file:
+        with open(file_path, "w") as json_file:
             json.dump(data, json_file, sort_keys=True, indent=4)
-            json_file.write('\n')
+            json_file.write("\n")
 
     @classmethod
     def walk(cls, drift_detection_directory):