cycode 3.24.1.dev2__tar.gz → 3.24.2__tar.gz

{cycode-3.24.1.dev2 → cycode-3.24.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cycode
-Version: 3.24.1.dev2
+Version: 3.24.2
 Summary: Boost security in your dev lifecycle via SAST, SCA, Secrets & IaC scanning.
 License-Expression: MIT
 License-File: LICENCE

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/__init__.py

@@ -5,4 +5,4 @@ import time as _time
 # end-to-end scan duration from the moment the user actually triggered it.
 _BOOT_WALL: float = _time.time()
 
-__version__ = '3.24.1.dev2'  # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag
+__version__ = '3.24.2'  # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/cli/apps/ai_guardrails/scan/guardrail_config.py

@@ -37,19 +37,11 @@ _KNOWN_GUARDRAIL_KEYS = frozenset(
     )
 )
 
-# CLI --ide names to matrix column names; identity for names not listed.
-_AGENT_BY_IDE_NAME = {'claude-code': 'claude'}
-
 
 def get_config_cache_path() -> Path:
     return Path.home() / CYCODE_CONFIGURATION_DIRECTORY / GUARDRAILS_CONFIG_FILE_NAME
 
 
-def agent_for_ide(ide_name: Optional[str]) -> str:
-    ide_name = (ide_name or '').lower()
-    return _AGENT_BY_IDE_NAME.get(ide_name, ide_name)
-
-
 def _default_sensitive_globs() -> list:
     return list(DEFAULT_SENSITIVE_PATH_GLOBS)
 
@@ -69,8 +61,9 @@ class GuardrailConfig:
         }
 
     def mode_for(self, guardrail_key: str, ide_name: Optional[str]) -> str:
+        """The platform keys the cells by our --ide names, so the lookup is direct."""
         agents = (self._guardrails.get(guardrail_key) or {}).get('agents') or {}
-        return str(agents.get(agent_for_ide(ide_name), GuardrailCellMode.REPORT.value)).lower()
+        return str(agents.get((ide_name or '').lower(), GuardrailCellMode.REPORT.value)).lower()
 
     def _modes_for_event(self, event_name: str, ide_name: Optional[str]) -> list:
         return [

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/cli/apps/ai_guardrails/scan/handlers.py

@@ -13,7 +13,7 @@ import os
 from dataclasses import dataclass
 from multiprocessing.pool import ThreadPool
 from multiprocessing.pool import TimeoutError as PoolTimeoutError
-from typing import TYPE_CHECKING, Callable, Optional
+from typing import TYPE_CHECKING, Callable, NamedTuple, Optional
 
 import typer
 
@@ -45,12 +45,32 @@ logger = get_logger('AI Guardrails')
 HandlerFn = Callable[[typer.Context, AIHookPayload, dict], HookDecision]
 
 
+class ScanOutcome(NamedTuple):
+    """What one guardrail scan came back with; the verdict is the server's, which applied the tenant's floors."""
+
+    violation_summary: Optional[str] = None
+    scan_id: Optional[str] = None
+    verdict: Optional[GuardrailsMode] = None
+
+
+NO_SCAN = ScanOutcome()
+
+
+def _parse_verdict(verdict: Optional[str]) -> Optional[GuardrailsMode]:
+    """The server spells the verdict "Block"/"Report" and omits it when the scan found nothing to decide on."""
+    if not verdict:
+        return None
+    try:
+        return GuardrailsMode(verdict.lower())
+    except ValueError:
+        logger.debug('Ignoring unknown guardrail verdict, %s', {'verdict': verdict})
+        return None
+
+
 def handle_before_submit_prompt(ctx: typer.Context, payload: AIHookPayload, policy: dict) -> HookDecision:
     """Scan prompt text for secrets before it's sent to the AI model."""
     ai_client = ctx.obj['ai_security_client']
 
-    prompt_config = get_policy_value(policy, 'prompt', default={})
-    effective_mode = get_effective_mode(prompt_config)
     prompt = payload.prompt or ''
     max_bytes = get_policy_value(policy, 'secrets', 'max_bytes', default=200000)
     timeout_ms = get_policy_value(policy, 'secrets', 'timeout_ms', default=30000)
@@ -62,20 +82,20 @@ def handle_before_submit_prompt(ctx: typer.Context, payload: AIHookPayload, poli
     error_message = None
 
     try:
-        violation_summary, scan_id = _scan_text_for_secrets(
+        scan_outcome = _scan_text_for_secrets(
             ctx,
             clipped,
             timeout_ms,
             payload=payload,
             event_type=AiHookEventType.PROMPT,
-            effective_mode=effective_mode,
         )
+        scan_id = scan_outcome.scan_id
 
-        if violation_summary:
+        if scan_outcome.violation_summary:
             block_reason = SECRETS_BLOCK_REASON_BY_EVENT_TYPE[AiHookEventType.PROMPT]
-            if effective_mode == GuardrailsMode.BLOCK:
+            if scan_outcome.verdict == GuardrailsMode.BLOCK:
                 outcome = AIHookOutcome.BLOCKED
-                user_message = f'Remove secrets before sending. {violation_summary}'
+                user_message = f'Remove secrets before sending. {scan_outcome.violation_summary}'
                 return HookDecision.deny(AiHookEventType.PROMPT, user_message)
             outcome = AIHookOutcome.WARNED
         return HookDecision.allow(AiHookEventType.PROMPT)
@@ -104,7 +124,6 @@ def handle_before_read_file(ctx: typer.Context, payload: AIHookPayload, policy:
     file_read_config = get_policy_value(policy, 'file_read', default={})
     file_path = payload.file_path or ''
     path_mode = get_effective_mode(file_read_config, action_key='path_action')
-    content_mode = get_effective_mode(file_read_config)
 
     scan_id = None
     block_reason = None
@@ -138,21 +157,20 @@ def handle_before_read_file(ctx: typer.Context, payload: AIHookPayload, policy:
                 outcome = AIHookOutcome.ALLOWED
 
         if get_policy_value(file_read_config, 'scan_content', default=True):
-            violation_summary, scan_id = _scan_path_for_secrets(
-                ctx, file_path, policy, payload=payload, effective_mode=content_mode
-            )
-            if violation_summary:
+            scan_outcome = _scan_path_for_secrets(ctx, file_path, policy, payload=payload)
+            scan_id = scan_outcome.scan_id
+            if scan_outcome.violation_summary:
                 block_reason = SECRETS_BLOCK_REASON_BY_EVENT_TYPE[AiHookEventType.FILE_READ]
-                if content_mode == GuardrailsMode.BLOCK:
+                if scan_outcome.verdict == GuardrailsMode.BLOCK:
                     outcome = AIHookOutcome.BLOCKED
-                    user_message = f'Cycode blocked reading {file_path}. {violation_summary}'
+                    user_message = f'Cycode blocked reading {file_path}. {scan_outcome.violation_summary}'
                     return HookDecision.deny(
                         AiHookEventType.FILE_READ,
                         user_message,
                         'Secrets detected; do not send this file to the model.',
                     )
                 outcome = AIHookOutcome.WARNED
-                user_message = f'Cycode detected secrets in {file_path}. {violation_summary}'
+                user_message = f'Cycode detected secrets in {file_path}. {scan_outcome.violation_summary}'
                 return HookDecision.ask(
                     AiHookEventType.FILE_READ,
                     user_message,
@@ -192,10 +210,9 @@ class _ArgScanFeature:
     """Configuration for a "scan some text and decide" event.
 
     MCP execution and command exec share identical scan-and-decide logic;
-    only the policy key, event type, and user-facing messages differ.
+    only the event type and user-facing messages differ.
     """
 
-    policy_key: str  # 'mcp' or 'command_exec'
     event_type: AiHookEventType
     deny_message: Callable[[str], str]
     deny_agent_message: str
@@ -213,11 +230,9 @@ def _handle_arg_scan(
     """Shared scan + decision flow for MCP_EXECUTION and COMMAND_EXEC events."""
     ai_client = ctx.obj['ai_security_client']
 
-    feature_config = get_policy_value(policy, feature.policy_key, default={})
     max_bytes = get_policy_value(policy, 'secrets', 'max_bytes', default=200000)
     timeout_ms = get_policy_value(policy, 'secrets', 'timeout_ms', default=30000)
     clipped = truncate_utf8(scan_text, max_bytes)
-    effective_mode = get_effective_mode(feature_config)
 
     scan_id = None
     block_reason = None
@@ -225,27 +240,27 @@ def _handle_arg_scan(
     error_message = None
 
     try:
-        violation_summary, scan_id = _scan_text_for_secrets(
+        scan_outcome = _scan_text_for_secrets(
             ctx,
             clipped,
             timeout_ms,
             payload=payload,
             event_type=feature.event_type,
-            effective_mode=effective_mode,
         )
-        if violation_summary:
+        scan_id = scan_outcome.scan_id
+        if scan_outcome.violation_summary:
             block_reason = SECRETS_BLOCK_REASON_BY_EVENT_TYPE[feature.event_type]
-            if effective_mode == GuardrailsMode.BLOCK:
+            if scan_outcome.verdict == GuardrailsMode.BLOCK:
                 outcome = AIHookOutcome.BLOCKED
                 return HookDecision.deny(
                     feature.event_type,
-                    feature.deny_message(violation_summary),
+                    feature.deny_message(scan_outcome.violation_summary),
                     feature.deny_agent_message,
                 )
             outcome = AIHookOutcome.WARNED
             return HookDecision.ask(
                 feature.event_type,
-                feature.ask_message(violation_summary),
+                feature.ask_message(scan_outcome.violation_summary),
                 feature.ask_agent_message,
             )
 
@@ -278,7 +293,6 @@ def handle_before_mcp_execution(ctx: typer.Context, payload: AIHookPayload, poli
         payload,
         policy,
         _ArgScanFeature(
-            policy_key='mcp',
             event_type=AiHookEventType.MCP_EXECUTION,
             deny_message=lambda v: f'Cycode blocked MCP tool call "{tool}". {v}',
             deny_agent_message='Do not pass secrets to tools. Use secret references (name/id) instead.',
@@ -329,11 +343,9 @@ def build_ai_guardrails_scan_parameters(
     paths: Optional[tuple[str, ...]],
     payload: AIHookPayload,
     event_type: AiHookEventType,
-    effective_mode: GuardrailsMode,
 ) -> dict:
     scan_parameters = get_scan_parameters(ctx, paths)
     scan_parameters.setdefault('metadata', {})['ai_guardrails'] = {
-        'mode': effective_mode.value,
         'ide_provider': payload.ide_provider,
         'detection_source': SECRETS_BLOCK_REASON_BY_EVENT_TYPE[event_type].value,
         'device_id': get_serial_number(),
@@ -360,13 +372,13 @@ def _setup_scan_context(ctx: typer.Context) -> typer.Context:
 
 def _perform_scan(
     ctx: typer.Context, documents: list[Document], scan_parameters: dict, timeout_seconds: float
-) -> tuple[Optional[str], Optional[str]]:
-    """Run a scan on documents, returning (violation_summary, scan_id).
+) -> ScanOutcome:
+    """Run a scan on documents.
 
     Raises on scan failure / timeout so the fail-open policy can take over.
     """
     if not documents:
-        return None, None
+        return NO_SCAN
 
     scan_batch_thread_func = _get_scan_documents_thread_func(
         ctx, is_git_diff=False, is_commit_range=False, scan_parameters=scan_parameters
@@ -377,7 +389,7 @@ def _perform_scan(
     with ThreadPool(processes=1) as pool:
         result = pool.apply_async(scan_batch_thread_func, (documents,))
         try:
-            scan_id, error, local_scan_result = result.get(timeout=timeout_seconds)
+            _, error, local_scan_result = result.get(timeout=timeout_seconds)
         except PoolTimeoutError:
             logger.debug('Scan timed out after %s seconds', timeout_seconds)
             raise RuntimeError(f'Scan timed out after {timeout_seconds} seconds') from None
@@ -387,15 +399,14 @@ def _perform_scan(
         raise RuntimeError(error.message)
 
     if not local_scan_result:
-        return None, None
-
-    scan_id = local_scan_result.scan_id
+        return NO_SCAN
 
-    if local_scan_result.issue_detected:
-        violation_summary = build_violation_summary([local_scan_result])
-        return violation_summary, scan_id
-
-    return None, scan_id
+    violation_summary = build_violation_summary([local_scan_result]) if local_scan_result.issue_detected else None
+    return ScanOutcome(
+        violation_summary=violation_summary,
+        scan_id=local_scan_result.scan_id,
+        verdict=_parse_verdict(local_scan_result.verdict),
+    )
 
 
 def _scan_text_for_secrets(
@@ -404,16 +415,15 @@ def _scan_text_for_secrets(
     timeout_ms: int,
     payload: AIHookPayload,
     event_type: AiHookEventType,
-    effective_mode: GuardrailsMode,
-) -> tuple[Optional[str], Optional[str]]:
+) -> ScanOutcome:
     """Scan text content for secrets using Cycode CLI."""
     if not text:
-        return None, None
+        return NO_SCAN
 
     document = Document(path='prompt-content.txt', content=text, is_git_diff_format=False)
     scan_ctx = _setup_scan_context(ctx)
     timeout_seconds = timeout_ms / 1000.0
-    scan_parameters = build_ai_guardrails_scan_parameters(scan_ctx, None, payload, event_type, effective_mode)
+    scan_parameters = build_ai_guardrails_scan_parameters(scan_ctx, None, payload, event_type)
     return _perform_scan(scan_ctx, [document], scan_parameters, timeout_seconds)
 
 
@@ -422,15 +432,14 @@ def _scan_path_for_secrets(
     file_path: str,
     policy: dict,
     payload: AIHookPayload,
-    effective_mode: GuardrailsMode,
-) -> tuple[Optional[str], Optional[str]]:
+) -> ScanOutcome:
     """Scan a file path for secrets."""
     if not file_path or not os.path.isfile(file_path):
-        return None, None
+        return NO_SCAN
 
     if is_path_configured_in_exclusions(str(ScanTypeOption.SECRET), os.path.abspath(file_path)):
         logger.debug('Skipping scan; the path is in the ignore paths list, %s', {'file_path': file_path})
-        return None, None
+        return NO_SCAN
 
     max_bytes = get_policy_value(policy, 'secrets', 'max_bytes', default=200000)
 
@@ -442,7 +451,5 @@ def _scan_path_for_secrets(
 
     document = Document(path=os.path.basename(file_path), content=content, is_git_diff_format=False)
     scan_ctx = _setup_scan_context(ctx)
-    scan_parameters = build_ai_guardrails_scan_parameters(
-        scan_ctx, (file_path,), payload, AiHookEventType.FILE_READ, effective_mode
-    )
+    scan_parameters = build_ai_guardrails_scan_parameters(scan_ctx, (file_path,), payload, AiHookEventType.FILE_READ)
     return _perform_scan(scan_ctx, [document], scan_parameters, timeout_seconds)

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/cli/apps/scan/code_scanner.py

@@ -1,5 +1,6 @@
 import os
 import time
+import zipfile
 from platform import platform
 from typing import TYPE_CHECKING, Callable, Optional
 
@@ -23,6 +24,7 @@ from cycode.cli.files_collector.path_documents import get_relevant_documents
 from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed
 from cycode.cli.files_collector.zip_documents import zip_documents
 from cycode.cli.models import CliError, Document, LocalScanResult
+from cycode.cli.utils.host_info import is_64bit
 from cycode.cli.utils.path_utils import get_absolute_path, get_path_by_os
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
 from cycode.cli.utils.scan_batch import run_parallel_batched_scan
@@ -145,6 +147,7 @@ def _get_scan_documents_thread_func(
     is_git_diff: bool,
     is_commit_range: bool,
     scan_parameters: dict,
+    prezipped: Optional['InMemoryZip'] = None,
 ) -> Callable[[list[Document]], tuple[str, CliError, LocalScanResult]]:
     cycode_client = ctx.obj['client']
     scan_type = ctx.obj['scan_type']
@@ -164,9 +167,14 @@ def _get_scan_documents_thread_func(
 
         should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option)
 
+        # the single ZIP flow already built the archive to check that it fits; don't build it twice
+        zipped_documents = prezipped
+
         try:
-            logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)})
-            zipped_documents = zip_documents(scan_type, batch)
+            if zipped_documents is None:
+                logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)})
+                zipped_documents = zip_documents(scan_type, batch)
+
             zip_file_size = zipped_documents.size
             scan_result = _perform_scan(
                 cycode_client,
@@ -189,6 +197,9 @@ def _get_scan_documents_thread_func(
         except Exception as e:
             error = handle_scan_exception(ctx, e, return_exception=True)
             error_message = str(e)
+        finally:
+            if zipped_documents is not None:
+                zipped_documents.cleanup()
 
         if local_scan_result:
             detections_count = local_scan_result.detections_count
@@ -225,34 +236,77 @@ def _get_scan_documents_thread_func(
     return _scan_batch_thread_func
 
 
+def _log_selected_upload_mode(mode: str, reason: str, documents_count: int) -> None:
+    logger.debug(
+        'Selected upload mode, %s',
+        {
+            'mode': mode,
+            'reason': reason,
+            'documents_count': documents_count,
+            'max_files_count': consts.ZIP_MAX_FILES_COUNT,
+            'zip64_enabled': is_64bit(),
+        },
+    )
+
+
+def _exceeds_non_zip64_files_count(documents_to_scan: list[Document]) -> bool:
+    """Whether a single ZIP can't hold all the documents because ZIP64 is unavailable.
+
+    Without ZIP64 (32-bit interpreter) the archive is capped at 65,535 entries.
+    """
+    return not is_64bit() and len(documents_to_scan) > consts.ZIP_MAX_FILES_COUNT
+
+
 def _run_presigned_upload_scan(
-    scan_batch_thread_func: Callable,
-    scan_type: str,
+    ctx: typer.Context,
+    is_git_diff: bool,
+    is_commit_range: bool,
+    scan_parameters: dict,
     documents_to_scan: list[Document],
     progress_bar: 'BaseProgressBar',
     printer: 'ConsolePrinter',
 ) -> tuple:
-    try:
-        # Try to zip all documents as a single batch; ZipTooLargeError raised if it exceeds the scan type's limit
-        zip_documents(scan_type, documents_to_scan)
-        # It fits: skip batching and upload everything as one ZIP
+    scan_type = ctx.obj['scan_type']
+    documents_count = len(documents_to_scan)
+
+    def run_batched() -> tuple:
         return run_parallel_batched_scan(
-            scan_batch_thread_func,
+            _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters),
             scan_type,
             documents_to_scan,
             progress_bar=progress_bar,
-            skip_batching=True,
         )
-    except custom_exceptions.ZipTooLargeError:
+
+    if _exceeds_non_zip64_files_count(documents_to_scan):
+        # Don't waste time zipping documents we already know won't fit into a single ZIP
+        _log_selected_upload_mode('batched', 'files_count_exceeds_non_zip64_limit', documents_count)
+        return run_batched()
+
+    zipped_documents = None
+    try:
+        # Try to zip all documents as a single batch; ZipTooLargeError raised if it exceeds the scan type's limit
+        zipped_documents = zip_documents(scan_type, documents_to_scan)
+    except (custom_exceptions.ZipTooLargeError, zipfile.LargeZipFile):
+        # LargeZipFile is a safety net: the files count pre-check above should have caught it already
+        _log_selected_upload_mode('batched', 'zip_too_large', documents_count)
+        if zipped_documents is not None:
+            zipped_documents.cleanup()
+
         printer.print_warning(
             'The scan is too large to upload as a single file. This may result in corrupted scan results.'
         )
-        return run_parallel_batched_scan(
-            scan_batch_thread_func,
-            scan_type,
-            documents_to_scan,
-            progress_bar=progress_bar,
-        )
+        return run_batched()
+
+    # It fits: skip batching and upload everything as one ZIP. The archive we just built is the one
+    # that gets uploaded, so the scan doesn't pay for compressing every document twice
+    _log_selected_upload_mode('single_zip', 'fits_single_zip', documents_count)
+    return run_parallel_batched_scan(
+        _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters, zipped_documents),
+        scan_type,
+        documents_to_scan,
+        progress_bar=progress_bar,
+        skip_batching=True,
+    )
 
 
 def scan_documents(
@@ -277,18 +331,19 @@ def scan_documents(
         )
         return
 
-    scan_batch_thread_func = _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters)
-
     # Presigned single-file upload is async-only; a --sync scan must stay on the batched inline path
     # so it never builds one oversized zip to POST synchronously.
     should_use_sync_flow = _should_use_sync_flow(ctx.info_name, scan_type, ctx.obj['sync'])
     if should_use_presigned_upload(scan_type) and not should_use_sync_flow:
         errors, local_scan_results = _run_presigned_upload_scan(
-            scan_batch_thread_func, scan_type, documents_to_scan, progress_bar, printer
+            ctx, is_git_diff, is_commit_range, scan_parameters, documents_to_scan, progress_bar, printer
         )
     else:
         errors, local_scan_results = run_parallel_batched_scan(
-            scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar
+            _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters),
+            scan_type,
+            documents_to_scan,
+            progress_bar=progress_bar,
         )
 
     try_set_aggregation_report_url_if_needed(ctx, scan_parameters, ctx.obj['client'], scan_type)

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/cli/apps/scan/commit_range_scanner.py

@@ -214,6 +214,9 @@ def _scan_commit_range_documents(
 
     zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size
 
+    from_commit_zipped_documents.cleanup()
+    to_commit_zipped_documents.cleanup()
+
     detections_count = relevant_detections_count = 0
     if local_scan_result:
         detections_count = local_scan_result.detections_count

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/cli/apps/scan/scan_result.py

@@ -79,6 +79,7 @@ def create_local_scan_result(
         issue_detected=len(relevant_document_detections_list) > 0,
         detections_count=detections_count,
         relevant_detections_count=relevant_detections_count,
+        verdict=scan_result.verdict,
     )
 
 
@@ -170,6 +171,7 @@ def get_sync_scan_result(scan_type: str, scan_results: 'ScanResultsSyncFlow') ->
         did_detect=True,
         detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_results.detection_messages),
         scan_id=scan_results.id,
+        verdict=scan_results.verdict,
     )
 
 

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/cli/consts.py

@@ -227,6 +227,12 @@ FILE_MAX_SIZE_LIMIT_IN_BYTES = 5000000
 PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 5 * 1024 * 1024 * 1024  # 5 GB (S3 presigned POST limit)
 PRESIGNED_UPLOAD_SCAN_TYPES = {SAST_SCAN_TYPE, SECRET_SCAN_TYPE}
 
+# the non-ZIP64 central directory stores the entry count in 16 bits; ZIP64 (64-bit interpreters) lifts it
+ZIP_MAX_FILES_COUNT = 65_535
+
+# the ZIP is built in memory up to this size, and spilled to a temp file beyond it
+ZIP_SPOOL_MAX_SIZE_IN_BYTES = 64 * 1024 * 1024
+
 DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 20 * 1024 * 1024
 ZIP_MAX_SIZE_LIMIT_IN_BYTES = {
     SCA_SCAN_TYPE: 200 * 1024 * 1024,

{cycode-3.24.1.dev2 → cycode-3.24.2}/cycode/cli/exceptions/handle_scan_errors.py

@@ -1,3 +1,4 @@
+import zipfile
 from typing import Optional
 
 import typer
@@ -26,6 +27,13 @@ def handle_scan_exception(ctx: typer.Context, err: Exception, *, return_exceptio
             'Please try ignoring irrelevant paths using the `cycode ignore --by-path` command '
             'and execute the scan again',
         ),
+        zipfile.LargeZipFile: CliError(
+            soft_fail=True,
+            code='zip_too_large_error',
+            message='The path you attempted to scan contains too many files to pack into a single archive. '
+            'Scanning such paths requires a 64-bit Python interpreter. '
+            'Please try ignoring irrelevant paths using a .cycodeignore file and execute the scan again',
+        ),
         custom_exceptions.FileCollectionError: CliError(
             soft_fail=False,
             code='file_collection_error',

cycode-3.24.2/cycode/cli/files_collector/models/in_memory_zip.py

@@ -0,0 +1,121 @@
+import shutil
+import tempfile
+from collections import defaultdict
+from os import SEEK_END
+from pathlib import Path
+from typing import IO, Optional
+from zipfile import ZIP_DEFLATED, ZipFile
+
+from cycode.cli import consts
+from cycode.cli.user_settings.configuration_manager import ConfigurationManager
+from cycode.cli.utils.host_info import is_64bit
+from cycode.cli.utils.path_utils import concat_unique_id
+from cycode.logger import get_logger
+
+logger = get_logger('ZIP')
+
+_SPOOL_DIRECTORY_NAME = 'tmp'
+
+
+def _get_spool_directory(configuration_manager: ConfigurationManager) -> Optional[str]:
+    """Directory to spill big ZIPs into. None falls back to the system temp directory."""
+    try:
+        directory = Path(configuration_manager.global_config_file_manager.get_config_directory_path())
+        spool_directory = directory / _SPOOL_DIRECTORY_NAME
+        spool_directory.mkdir(parents=True, exist_ok=True)
+        return str(spool_directory)
+    except OSError as e:
+        logger.debug('Failed to create the spool directory; falling back to the system one', exc_info=e)
+        return None
+
+
+class InMemoryZip:
+    def __init__(self) -> None:
+        self.configuration_manager = ConfigurationManager()
+
+        self._spool_max_size = consts.ZIP_SPOOL_MAX_SIZE_IN_BYTES
+        self._buffer = tempfile.SpooledTemporaryFile(  # noqa: SIM115  # closed by cleanup(), lives past close()
+            max_size=self._spool_max_size,
+            dir=_get_spool_directory(self.configuration_manager),
+        )
+
+        # ZIP64 lifts the 65,535 entries and 4 GiB caps of the original ZIP format.
+        # It requires 64-bit offsets, so we only enable it on a 64-bit interpreter.
+        self._allow_zip64 = is_64bit()
+        self.zip = ZipFile(self._buffer, mode='a', compression=ZIP_DEFLATED, allowZip64=self._allow_zip64)
+
+        self._files_count = 0
+        self._extension_statistics = defaultdict(int)
+
+    def append(self, filename: str, unique_id: Optional[str], content: str) -> None:
+        self._files_count += 1
+        self._extension_statistics[Path(filename).suffix] += 1
+
+        if unique_id:
+            filename = concat_unique_id(filename, unique_id)
+
+        # Encode content to bytes with error handling to handle surrogate characters
+        # that cannot be encoded to UTF-8. Use 'replace' to replace invalid characters
+        # with the Unicode replacement character (U+FFFD).
+        content_bytes = content.encode('utf-8', errors='replace')
+        self.zip.writestr(filename, content_bytes)
+
+    def close(self) -> None:
+        self.zip.close()
+
+    def cleanup(self) -> None:
+        """Release the buffer, deleting the spilled temp file if there is one."""
+        self._buffer.close()
+
+    def __enter__(self) -> 'InMemoryZip':  # noqa: PYI034  # typing.Self needs Python 3.11
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.cleanup()
+
+    def stream(self) -> IO[bytes]:
+        """The whole archive as a file object, rewound. Doesn't copy it into memory.
+
+        Note: before Python 3.11 SpooledTemporaryFile isn't a real IOBase, so the returned object
+        has no seekable()/readable()/writable(). read/seek/tell work on every supported version.
+        """
+        self._buffer.seek(0)
+        return self._buffer
+
+    def read(self) -> bytes:
+        self._buffer.seek(0)
+        return self._buffer.read()
+
+    def write_on_disk(self, path: 'Path') -> None:
+        with open(path, 'wb') as f:
+            shutil.copyfileobj(self.stream(), f)
+
+    @property
+    def size(self) -> int:
+        position = self._buffer.tell()
+        try:
+            self._buffer.seek(0, SEEK_END)
+            return self._buffer.tell()
+        finally:
+            self._buffer.seek(position)
+
+    @property
+    def is_rolled_over(self) -> bool:
+        """Whether the archive outgrew the threshold and moved from memory to the disk.
+
+        SpooledTemporaryFile spills on the write that crosses max_size, and the archive only grows,
+        so the size says it without reaching into the private _rolled flag.
+        """
+        return self.size > self._spool_max_size
+
+    @property
+    def allow_zip64(self) -> bool:
+        return self._allow_zip64
+
+    @property
+    def files_count(self) -> int:
+        return self._files_count
+
+    @property
+    def extension_statistics(self) -> dict[str, int]:
+        return dict(self._extension_statistics)