cycode 3.0.1.dev1__tar.gz → 3.0.2.dev2__tar.gz

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cycode
-Version: 3.0.1.dev1
+Version: 3.0.2.dev2
 Summary: Boost security in your dev lifecycle via SAST, SCA, Secrets & IaC scanning.
 Home-page: https://github.com/cycodehq/cycode-cli
 License: MIT

cycode-3.0.2.dev2/cycode/__init__.py

@@ -0,0 +1 @@
+__version__ = '3.0.2.dev2'  # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/apps/scan/code_scanner.py

@@ -15,7 +15,7 @@ from cycode.cli.config import configuration_manager
 from cycode.cli.console import console
 from cycode.cli.exceptions import custom_exceptions
 from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception
-from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan
+from cycode.cli.files_collector.excluder import excluder
 from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip
 from cycode.cli.files_collector.path_documents import get_relevant_documents
 from cycode.cli.files_collector.repository_documents import (
@@ -56,8 +56,8 @@ def scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
         progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES,
         repo_path=repo_path,
     )
-    git_head_documents = exclude_irrelevant_documents_to_scan(scan_type, git_head_documents)
-    pre_committed_documents = exclude_irrelevant_documents_to_scan(scan_type, pre_committed_documents)
+    git_head_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, git_head_documents)
+    pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, pre_committed_documents)
     sca_code_scanner.perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
     scan_commit_range_documents(
         ctx,
@@ -77,8 +77,8 @@ def scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str) -> N
     from_commit_documents, to_commit_documents = get_commit_range_modified_documents(
         progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, path, from_commit_rev, to_commit_rev
     )
-    from_commit_documents = exclude_irrelevant_documents_to_scan(scan_type, from_commit_documents)
-    to_commit_documents = exclude_irrelevant_documents_to_scan(scan_type, to_commit_documents)
+    from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, from_commit_documents)
+    to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, to_commit_documents)
     sca_code_scanner.perform_pre_commit_range_scan_actions(
         path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev
     )
@@ -288,7 +288,7 @@ def scan_commit_range(
             {'path': path, 'commit_range': commit_range, 'commit_id': commit_id},
         )
 
-        documents_to_scan.extend(exclude_irrelevant_documents_to_scan(scan_type, commit_documents_to_scan))
+        documents_to_scan.extend(excluder.exclude_irrelevant_documents_to_scan(scan_type, commit_documents_to_scan))
 
     logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan})
     logger.debug('Starting to scan commit range (it may take a few minutes)')

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/apps/scan/pre_commit/pre_commit_command.py

@@ -5,7 +5,7 @@ import typer
 
 from cycode.cli import consts
 from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents, scan_sca_pre_commit
-from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan
+from cycode.cli.files_collector.excluder import excluder
 from cycode.cli.files_collector.repository_documents import (
     get_diff_file_content,
     get_diff_file_path,
@@ -45,5 +45,5 @@ def pre_commit_command(
         progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES)
         documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file)))
 
-    documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan)
+    documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan)
     scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True)

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/apps/scan/repository/repository_command.py

@@ -7,7 +7,7 @@ import typer
 from cycode.cli import consts
 from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents
 from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception
-from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan
+from cycode.cli.files_collector.excluder import excluder
 from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries
 from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions
 from cycode.cli.logger import logger
@@ -57,7 +57,7 @@ def repository_command(
                 )
             )
 
-        documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan)
+        documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan)
 
         perform_pre_scan_documents_actions(ctx, scan_type, documents_to_scan)
 

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/apps/scan/scan_command.py

@@ -9,6 +9,7 @@ from cycode.cli.consts import (
     ISSUE_DETECTED_STATUS_CODE,
     NO_ISSUES_STATUS_CODE,
 )
+from cycode.cli.files_collector.excluder import excluder
 from cycode.cli.utils import scan_utils
 from cycode.cli.utils.get_api_client import get_scan_cycode_client
 from cycode.cli.utils.sentry import add_breadcrumb
@@ -138,13 +139,19 @@ def scan_command(
 
     ctx.obj['show_secret'] = show_secret
     ctx.obj['soft_fail'] = soft_fail
-    ctx.obj['client'] = get_scan_cycode_client(ctx)
     ctx.obj['scan_type'] = scan_type
     ctx.obj['sync'] = sync
     ctx.obj['severity_threshold'] = severity_threshold
     ctx.obj['monitor'] = monitor
     ctx.obj['report'] = report
 
+    scan_client = get_scan_cycode_client(ctx)
+    ctx.obj['client'] = scan_client
+
+    remote_scan_config = scan_client.get_scan_configuration_safe(scan_type)
+    if remote_scan_config:
+        excluder.apply_scan_config(str(scan_type), remote_scan_config)
+
     if export_type and export_file:
         console_printer = ctx.obj['console_printer']
         console_printer.enable_recording(export_type, export_file)

cycode-3.0.2.dev2/cycode/cli/files_collector/excluder.py

@@ -0,0 +1,174 @@
+from typing import TYPE_CHECKING
+
+from cycode.cli import consts
+from cycode.cli.config import configuration_manager
+from cycode.cli.user_settings.config_file_manager import ConfigFileManager
+from cycode.cli.utils.path_utils import get_file_size, is_binary_file, is_sub_path
+from cycode.cli.utils.string_utils import get_content_size, is_binary_content
+from cycode.logger import get_logger
+
+if TYPE_CHECKING:
+    from cycode.cli.models import Document
+    from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection
+    from cycode.cyclient import models
+
+
+logger = get_logger('File Excluder')
+
+
+def _is_subpath_of_cycode_configuration_folder(filename: str) -> bool:
+    return (
+        is_sub_path(configuration_manager.global_config_file_manager.get_config_directory_path(), filename)
+        or is_sub_path(configuration_manager.local_config_file_manager.get_config_directory_path(), filename)
+        or filename.endswith(ConfigFileManager.get_config_file_route())
+    )
+
+
+def _is_path_configured_in_exclusions(scan_type: str, file_path: str) -> bool:
+    exclusions_by_path = configuration_manager.get_exclusions_by_scan_type(scan_type).get(
+        consts.EXCLUSIONS_BY_PATH_SECTION_NAME, []
+    )
+    return any(is_sub_path(exclusion_path, file_path) for exclusion_path in exclusions_by_path)
+
+
+def _does_file_exceed_max_size_limit(filename: str) -> bool:
+    return get_file_size(filename) > consts.FILE_MAX_SIZE_LIMIT_IN_BYTES
+
+
+def _does_document_exceed_max_size_limit(content: str) -> bool:
+    return get_content_size(content) > consts.FILE_MAX_SIZE_LIMIT_IN_BYTES
+
+
+def _is_file_relevant_for_sca_scan(filename: str) -> bool:
+    if any(sca_excluded_path in filename for sca_excluded_path in consts.SCA_EXCLUDED_PATHS):
+        logger.debug(
+            'The file is irrelevant because it is from the inner path of node_modules, %s', {'filename': filename}
+        )
+        return False
+
+    return True
+
+
+class Excluder:
+    def __init__(self) -> None:
+        self._scannable_extensions: dict[str, tuple[str, ...]] = {
+            consts.IAC_SCAN_TYPE: consts.IAC_SCAN_SUPPORTED_FILES,
+            consts.SCA_SCAN_TYPE: consts.SCA_CONFIGURATION_SCAN_SUPPORTED_FILES,
+        }
+        self._non_scannable_extensions: dict[str, tuple[str, ...]] = {
+            consts.SECRET_SCAN_TYPE: consts.SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE,
+        }
+
+    def apply_scan_config(self, scan_type: str, scan_config: 'models.ScanConfiguration') -> None:
+        if scan_config.scannable_extensions:
+            self._scannable_extensions[scan_type] = tuple(scan_config.scannable_extensions)
+
+    def _is_file_extension_supported(self, scan_type: str, filename: str) -> bool:
+        filename = filename.lower()
+
+        scannable_extensions = self._scannable_extensions.get(scan_type)
+        if scannable_extensions:
+            return filename.endswith(scannable_extensions)
+
+        non_scannable_extensions = self._non_scannable_extensions.get(scan_type)
+        if non_scannable_extensions:
+            return not filename.endswith(non_scannable_extensions)
+
+        return True
+
+    def _is_relevant_file_to_scan_common(self, scan_type: str, filename: str) -> bool:
+        if _is_subpath_of_cycode_configuration_folder(filename):
+            logger.debug(
+                'The document is irrelevant because it is in the Cycode configuration directory, %s',
+                {'filename': filename, 'configuration_directory': consts.CYCODE_CONFIGURATION_DIRECTORY},
+            )
+            return False
+
+        if _is_path_configured_in_exclusions(scan_type, filename):
+            logger.debug(
+                'The document is irrelevant because its path is in the ignore paths list, %s', {'filename': filename}
+            )
+            return False
+
+        if not self._is_file_extension_supported(scan_type, filename):
+            logger.debug(
+                'The document is irrelevant because its extension is not supported, %s',
+                {'scan_type': scan_type, 'filename': filename},
+            )
+            return False
+
+        return True
+
+    def _is_relevant_file_to_scan(self, scan_type: str, filename: str) -> bool:
+        if not self._is_relevant_file_to_scan_common(scan_type, filename):
+            return False
+
+        if is_binary_file(filename):
+            logger.debug('The file is irrelevant because it is a binary file, %s', {'filename': filename})
+            return False
+
+        if scan_type != consts.SCA_SCAN_TYPE and _does_file_exceed_max_size_limit(filename):
+            logger.debug(
+                'The file is irrelevant because it has exceeded the maximum size limit, %s',
+                {
+                    'max_file_size': consts.FILE_MAX_SIZE_LIMIT_IN_BYTES,
+                    'file_size': get_file_size(filename),
+                    'filename': filename,
+                },
+            )
+            return False
+
+        return not (scan_type == consts.SCA_SCAN_TYPE and not _is_file_relevant_for_sca_scan(filename))
+
+    def _is_relevant_document_to_scan(self, scan_type: str, filename: str, content: str) -> bool:
+        if not self._is_relevant_file_to_scan_common(scan_type, filename):
+            return False
+
+        if is_binary_content(content):
+            logger.debug('The document is irrelevant because it is a binary file, %s', {'filename': filename})
+            return False
+
+        if scan_type != consts.SCA_SCAN_TYPE and _does_document_exceed_max_size_limit(content):
+            logger.debug(
+                'The document is irrelevant because it has exceeded the maximum size limit, %s',
+                {
+                    'max_document_size': consts.FILE_MAX_SIZE_LIMIT_IN_BYTES,
+                    'document_size': get_content_size(content),
+                    'filename': filename,
+                },
+            )
+            return False
+
+        return True
+
+    def exclude_irrelevant_files(
+        self,
+        progress_bar: 'BaseProgressBar',
+        progress_bar_section: 'ProgressBarSection',
+        scan_type: str,
+        filenames: list[str],
+    ) -> list[str]:
+        relevant_files = []
+        for filename in filenames:
+            progress_bar.update(progress_bar_section)
+            if self._is_relevant_file_to_scan(scan_type, filename):
+                relevant_files.append(filename)
+
+        is_sub_path.cache_clear()  # free up memory
+
+        return relevant_files
+
+    def exclude_irrelevant_documents_to_scan(
+        self, scan_type: str, documents_to_scan: list['Document']
+    ) -> list['Document']:
+        logger.debug('Excluding irrelevant documents to scan')
+
+        relevant_documents = []
+        for document in documents_to_scan:
+            if self._is_relevant_document_to_scan(scan_type, document.path, document.content):
+                relevant_documents.append(document)
+
+        return relevant_documents
+
+
+excluder = Excluder()

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/models/in_memory_zip.py

@@ -1,25 +1,28 @@
+from collections import defaultdict
 from io import BytesIO
+from pathlib import Path
 from sys import getsizeof
-from typing import TYPE_CHECKING, Optional
+from typing import Optional
 from zipfile import ZIP_DEFLATED, ZipFile
 
 from cycode.cli.user_settings.configuration_manager import ConfigurationManager
 from cycode.cli.utils.path_utils import concat_unique_id
 
-if TYPE_CHECKING:
-    from pathlib import Path
-
 
 class InMemoryZip:
     def __init__(self) -> None:
         self.configuration_manager = ConfigurationManager()
 
-        # Create the in-memory file-like object
         self.in_memory_zip = BytesIO()
-        self.zip = ZipFile(self.in_memory_zip, 'a', ZIP_DEFLATED, False)
+        self.zip = ZipFile(self.in_memory_zip, mode='a', compression=ZIP_DEFLATED, allowZip64=False)
+
+        self._files_count = 0
+        self._extension_statistics = defaultdict(int)
 
     def append(self, filename: str, unique_id: Optional[str], content: str) -> None:
-        # Write the file to the in-memory zip
+        self._files_count += 1
+        self._extension_statistics[Path(filename).suffix] += 1
+
         if unique_id:
             filename = concat_unique_id(filename, unique_id)
 
@@ -28,7 +31,6 @@ class InMemoryZip:
     def close(self) -> None:
         self.zip.close()
 
-    # to bytes
     def read(self) -> bytes:
         self.in_memory_zip.seek(0)
         return self.in_memory_zip.read()
@@ -40,3 +42,11 @@ class InMemoryZip:
     @property
     def size(self) -> int:
         return getsizeof(self.in_memory_zip)
+
+    @property
+    def files_count(self) -> int:
+        return self._files_count
+
+    @property
+    def extension_statistics(self) -> dict[str, int]:
+        return dict(self._extension_statistics)

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/path_documents.py

@@ -1,7 +1,7 @@
 import os
 from typing import TYPE_CHECKING
 
-from cycode.cli.files_collector.excluder import exclude_irrelevant_files
+from cycode.cli.files_collector.excluder import excluder
 from cycode.cli.files_collector.iac.tf_content_generator import (
     generate_tf_content_from_tfplan,
     generate_tfplan_document_name,
@@ -54,7 +54,9 @@ def _get_relevant_files(
     progress_bar_section_len = len(all_files_to_scan) * 2
     progress_bar.set_section_length(progress_bar_section, progress_bar_section_len)
 
-    relevant_files_to_scan = exclude_irrelevant_files(progress_bar, progress_bar_section, scan_type, all_files_to_scan)
+    relevant_files_to_scan = excluder.exclude_irrelevant_files(
+        progress_bar, progress_bar_section, scan_type, all_files_to_scan
+    )
 
     # after finishing the first processing (excluding),
     # we must update the progress bar stage with respect of excluded files.

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/sca/base_restore_dependencies.py

@@ -59,14 +59,13 @@ class BaseRestoreDependencies(ABC):
         manifest_file_path = self.get_manifest_file_path(document)
         restore_file_path = build_dep_tree_path(document.absolute_path, self.get_lock_file_name())
         relative_restore_file_path = build_dep_tree_path(document.path, self.get_lock_file_name())
-        working_directory_path = self.get_working_directory(document)
 
         if not self.verify_restore_file_already_exist(restore_file_path):
             output = execute_commands(
-                self.get_commands(manifest_file_path),
-                self.command_timeout,
+                commands=self.get_commands(manifest_file_path),
+                timeout=self.command_timeout,
                 output_file_path=restore_file_path if self.create_output_file_manually else None,
-                working_directory=working_directory_path,
+                working_directory=self.get_working_directory(document),
             )
             if output is None:  # one of the commands failed
                 return None
@@ -75,7 +74,7 @@ class BaseRestoreDependencies(ABC):
         return Document(relative_restore_file_path, restore_file_content, self.is_git_diff)
 
     def get_working_directory(self, document: Document) -> Optional[str]:
-        return None
+        return os.path.dirname(document.absolute_path)
 
     @staticmethod
     def verify_restore_file_already_exist(restore_file_path: str) -> bool:

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/sca/go/restore_go_dependencies.py

@@ -43,6 +43,3 @@ class RestoreGoDependencies(BaseRestoreDependencies):
 
     def get_lock_file_name(self) -> str:
         return GO_RESTORE_FILE_NAME
-
-    def get_working_directory(self, document: Document) -> Optional[str]:
-        return os.path.dirname(document.absolute_path)

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py

@@ -30,34 +30,36 @@ class RestoreMavenDependencies(BaseRestoreDependencies):
         return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME)
 
     def try_restore_dependencies(self, document: Document) -> Optional[Document]:
-        restore_dependencies_document = super().try_restore_dependencies(document)
         manifest_file_path = self.get_manifest_file_path(document)
         if document.content is None:
-            restore_dependencies_document = self.restore_from_secondary_command(
-                document, manifest_file_path, restore_dependencies_document
-            )
-        else:
-            restore_dependencies_document.content = get_file_content(
-                join_paths(get_file_dir(manifest_file_path), self.get_lock_file_name())
-            )
+            return self.restore_from_secondary_command(document, manifest_file_path)
+
+        restore_dependencies_document = super().try_restore_dependencies(document)
+        if restore_dependencies_document is None:
+            return None
+
+        restore_dependencies_document.content = get_file_content(
+            join_paths(get_file_dir(manifest_file_path), self.get_lock_file_name())
+        )
 
         return restore_dependencies_document
 
-    def restore_from_secondary_command(
-        self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document]
-    ) -> Optional[Document]:
-        # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg
-        secondary_restore_command = create_secondary_restore_commands(manifest_file_path)
-        backup_restore_content = execute_commands(secondary_restore_command, self.command_timeout)
-        restore_dependencies_document = Document(
-            build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME), backup_restore_content, self.is_git_diff
+    def restore_from_secondary_command(self, document: Document, manifest_file_path: str) -> Optional[Document]:
+        restore_content = execute_commands(
+            commands=create_secondary_restore_commands(manifest_file_path),
+            timeout=self.command_timeout,
+            working_directory=self.get_working_directory(document),
         )
-        restore_dependencies = None
-        if restore_dependencies_document.content is not None:
-            restore_dependencies = restore_dependencies_document
-            restore_dependencies.content = get_file_content(MAVEN_DEP_TREE_FILE_NAME)
+        if restore_content is None:
+            return None
 
-        return restore_dependencies
+        restore_file_path = build_dep_tree_path(document.absolute_path, MAVEN_DEP_TREE_FILE_NAME)
+        return Document(
+            path=build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME),
+            content=get_file_content(restore_file_path),
+            is_git_diff_format=self.is_git_diff,
+            absolute_path=restore_file_path,
+        )
 
 
 def create_secondary_restore_commands(manifest_file_path: str) -> list[list[str]]:

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py

@@ -1,6 +1,3 @@
-import os
-from typing import Optional
-
 from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
 from cycode.cli.models import Document
 
@@ -17,6 +14,3 @@ class RestoreRubyDependencies(BaseRestoreDependencies):
 
     def get_lock_file_name(self) -> str:
         return RUBY_LOCK_FILE_NAME
-
-    def get_working_directory(self, document: Document) -> Optional[str]:
-        return os.path.dirname(document.absolute_path)

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py

@@ -1,6 +1,3 @@
-import os
-from typing import Optional
-
 from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
 from cycode.cli.models import Document
 
@@ -17,6 +14,3 @@ class RestoreSbtDependencies(BaseRestoreDependencies):
 
     def get_lock_file_name(self) -> str:
         return SBT_LOCK_FILE_NAME
-
-    def get_working_directory(self, document: Document) -> Optional[str]:
-        return os.path.dirname(document.absolute_path)

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cli/files_collector/sca/sca_code_scanner.py

@@ -92,17 +92,16 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]:
 
 def try_restore_dependencies(
     ctx: typer.Context,
-    documents_to_add: dict[str, Document],
     restore_dependencies: 'BaseRestoreDependencies',
     document: Document,
-) -> None:
+) -> Optional[Document]:
     if not restore_dependencies.is_project(document):
-        return
+        return None
 
     restore_dependencies_document = restore_dependencies.restore(document)
     if restore_dependencies_document is None:
         logger.warning('Error occurred while trying to generate dependencies tree, %s', {'filename': document.path})
-        return
+        return None
 
     if restore_dependencies_document.content is None:
         logger.warning('Error occurred while trying to generate dependencies tree, %s', {'filename': document.path})
@@ -114,10 +113,7 @@ def try_restore_dependencies(
         manifest_file_path = get_manifest_file_path(document, is_monitor_action, project_path)
         logger.debug('Succeeded to generate dependencies tree on path: %s', manifest_file_path)
 
-    if restore_dependencies_document.path in documents_to_add:
-        logger.debug('Duplicate document on restore for path: %s', restore_dependencies_document.path)
-    else:
-        documents_to_add[restore_dependencies_document.path] = restore_dependencies_document
+    return restore_dependencies_document
 
 
 def add_dependencies_tree_document(
@@ -128,7 +124,14 @@ def add_dependencies_tree_document(
 
     for restore_dependencies in restore_dependencies_list:
         for document in documents_to_scan:
-            try_restore_dependencies(ctx, documents_to_add, restore_dependencies, document)
+            restore_dependencies_document = try_restore_dependencies(ctx, restore_dependencies, document)
+            if restore_dependencies_document is None:
+                continue
+
+            if restore_dependencies_document.path in documents_to_add:
+                logger.debug('Duplicate document on restore for path: %s', restore_dependencies_document.path)
+            else:
+                documents_to_add[restore_dependencies_document.path] = restore_dependencies_document
 
     # mutate original list using slice assignment
     documents_to_scan[:] = list(documents_to_add.values())

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cyclient/models.py

@@ -500,3 +500,19 @@ class SupportedModulesPreferencesSchema(Schema):
     @post_load
     def build_dto(self, data: dict[str, Any], **_) -> 'SupportedModulesPreferences':
         return SupportedModulesPreferences(**data)
+
+
+@dataclass
+class ScanConfiguration:
+    scannable_extensions: list[str]
+
+
+class ScanConfigurationSchema(Schema):
+    class Meta:
+        unknown = EXCLUDE
+
+    scannable_extensions = fields.List(fields.String(), allow_none=True)
+
+    @post_load
+    def build_dto(self, data: dict[str, Any], **_) -> 'ScanConfiguration':
+        return ScanConfiguration(**data)

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/cycode/cyclient/scan_client.py

@@ -1,16 +1,17 @@
 import json
 from copy import deepcopy
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Optional, Union
 from uuid import UUID
 
 from requests import Response
 
 from cycode.cli import consts
 from cycode.cli.config import configuration_manager
-from cycode.cli.exceptions.custom_exceptions import CycodeError
+from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestHttpError
 from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip
 from cycode.cyclient import models
 from cycode.cyclient.cycode_client_base import CycodeClientBase
+from cycode.cyclient.logger import logger
 
 if TYPE_CHECKING:
     from cycode.cyclient.scan_config_base import ScanConfigBase
@@ -100,12 +101,19 @@ class ScanClient:
         is_commit_range: bool = False,
     ) -> models.ScanInitializationResponse:
         files = {'file': ('multiple_files_scan.zip', zip_file.read())}
+
+        compression_manifest = {
+            'file_count_by_extension': zip_file.extension_statistics,
+            'file_count': zip_file.files_count,
+        }
+
         response = self.scan_cycode_client.post(
             url_path=self.get_zipped_file_scan_async_url_path(scan_type),
             data={
                 'is_git_diff': is_git_diff,
                 'scan_parameters': json.dumps(scan_parameters),
                 'is_commit_range': is_commit_range,
+                'compression_manifest': json.dumps(compression_manifest),
             },
             files=files,
         )
@@ -245,3 +253,25 @@ class ScanClient:
     @staticmethod
     def parse_scan_response(response: Response) -> models.ScanResult:
         return models.ScanResultSchema().load(response.json())
+
+    def get_scan_configuration_path(self, scan_type: str) -> str:
+        correct_scan_type = self.scan_config.get_async_scan_type(scan_type)
+        return f'{self.get_scan_service_url_path(scan_type)}/{correct_scan_type}/configuration'
+
+    def get_scan_configuration(self, scan_type: str) -> models.ScanConfiguration:
+        response = self.scan_cycode_client.get(
+            url_path=self.get_scan_configuration_path(scan_type),
+            hide_response_content_log=self._hide_response_log,
+        )
+        return models.ScanConfigurationSchema().load(response.json())
+
+    def get_scan_configuration_safe(self, scan_type: str) -> Optional['models.ScanConfiguration']:
+        try:
+            return self.get_scan_configuration(scan_type)
+        except RequestHttpError as e:
+            if e.status_code == 404:
+                logger.debug(
+                    'Remote scan configuration is not supported for this scan type: %s', {'scan_type': scan_type}
+                )
+            else:
+                logger.debug('Failed to get remote scan configuration: %s', {'scan_type': scan_type}, exc_info=e)

{cycode-3.0.1.dev1 → cycode-3.0.2.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cycode"
-version = "3.0.1.dev1" # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag
+version = "3.0.2.dev2" # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag
 description = "Boost security in your dev lifecycle via SAST, SCA, Secrets & IaC scanning."
 keywords=["secret-scan", "cycode", "devops", "token", "secret", "security", "cycode", "code"]
 authors = ["Cycode <support@cycode.com>"]

cycode-3.0.1.dev1/cycode/__init__.py

@@ -1 +0,0 @@
-__version__ = '3.0.1.dev1'  # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag