regscale-cli 6.16.1.0__py3-none-any.whl → 6.16.2.0__py3-none-any.whl

regscale/integrations/jsonl_scanner_integration.py

@@ -0,0 +1,869 @@
+"""
+Abstract base class for scanner integrations that use JSONL files for intermediate storage.
+"""
+
+import dataclasses
+import json
+import logging
+import os
+import shutil
+import tempfile
+from typing import Any, Dict, Iterator, Optional, Union, Tuple, TypeVar, Type, List
+
+import boto3
+from pathlib import Path
+
+from regscale.core.app.utils.file_utils import is_s3_path, read_file, find_files, download_from_s3
+from regscale.exceptions import ValidationException
+from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding, ScannerIntegration
+from regscale.models.app_models.mapping import Mapping
+
+logger = logging.getLogger("regscale")
+
+# Define generic types for items that can be written to file
+T = TypeVar("T")
+ItemType = TypeVar("ItemType", IntegrationAsset, IntegrationFinding)
+
+
+class JSONLScannerIntegration(ScannerIntegration):
+    """
+    Abstract base class for scanner integrations that use JSONL files for intermediate storage.
+
+    This class extends ScannerIntegration to provide common functionality for scanners
+    that process source files (local or S3) and store the results in JSONL files before syncing to RegScale.
+    Supports reading files directly without downloading when read_files_only is True.
+
+    Subclasses must implement:
+    - find_valid_files: To find valid source files
+    - parse_asset: To parse an asset from a source file
+    - parse_finding: To parse a finding from a source file
+    - is_valid_file: To validate a file before processing
+    """
+
+    # Constants for file paths - subclasses should override these
+    ASSETS_FILE = "./artifacts/assets.jsonl"
+    FINDINGS_FILE = "./artifacts/findings.jsonl"
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the JSONLScannerIntegration.
+        """
+        # Extract S3-related kwargs
+        self.s3_bucket = kwargs.pop("s3_bucket", None)
+        self.s3_prefix = kwargs.pop("s3_prefix", "")
+        self.aws_profile = kwargs.pop("aws_profile", "default")
+
+        self.plan_id = kwargs.pop("plan_id", None)
+        self.file_path = kwargs.pop("file_path", None)
+        self.empty_files: bool = True
+        self.scan_date = kwargs.pop("scan_date", None)
+        self.download_destination = kwargs.pop("destination", None)
+        self.file_pattern = kwargs.pop("file_pattern", "*.json")
+        self.read_files_only = kwargs.pop("read_files_only", False)
+
+        # Extract mapping-related kwargs
+        self.disable_mapping = kwargs.pop("disable_mapping", False)
+        self.mapping_path = kwargs.pop("mapping_path", f"./mappings/{self.__class__.__name__.lower()}/mapping.json")
+        self.required_asset_fields = kwargs.pop("required_asset_fields", ["identifier", "name"])
+        self.required_finding_fields = kwargs.pop("required_finding_fields", ["asset_identifier", "title", "severity"])
+        self.mapping = self._load_mapping() if not self.disable_mapping else None
+
+        # Initialize parent class
+        super().__init__(plan_id=self.plan_id, **kwargs)
+
+        self.s3_client = None
+        if self.s3_bucket and not self.read_files_only:
+            try:
+                session = boto3.Session(profile_name=self.aws_profile)
+                self.s3_client = session.client("s3")
+            except Exception as e:
+                logger.error(f"Failed to initialize S3 client with profile {self.aws_profile}: {str(e)}")
+                raise ValidationException(f"S3 client initialization failed: {str(e)}")
+
+    def _load_mapping(self) -> Optional[Mapping]:
+        """Load the mapping configuration from a JSON file."""
+        try:
+            mapping_file = Path(self.mapping_path)
+            if mapping_file.exists():
+                with mapping_file.open("r") as f:
+                    mapping_data = json.load(f)
+                    return Mapping(**mapping_data)
+            logger.debug(f"No mapping file found at {self.mapping_path}, using default mapping")
+            return None
+        except Exception as e:
+            logger.error(f"Error loading mapping file {self.mapping_path}: {str(e)}")
+            return None
+
+    def _apply_mapping(
+        self, source_data: Dict[str, Any], target_fields: Dict[str, Any], mapping: Dict[str, str]
+    ) -> Dict[str, Any]:
+        """Apply field mapping from source data to target fields."""
+        mapped_data = target_fields.copy()
+
+        if self.disable_mapping or not self.mapping or not hasattr(self.mapping, "fields"):
+            return {**source_data, **mapped_data}
+
+        for target_field, source_field in mapping.items():
+            if source_field in source_data:
+                mapped_data[target_field] = source_data[source_field]
+            elif isinstance(source_field, dict) and "default" in source_field:
+                mapped_data[target_field] = source_field["default"]
+
+        return mapped_data
+
+    def _validate_fields(self, item: Union[IntegrationAsset, IntegrationFinding], required_fields: list) -> None:
+        """Validate that all required fields are present and non-empty."""
+        missing_fields = []
+        item_dict = dataclasses.asdict(item)
+
+        for field in required_fields:
+            if field not in item_dict or not item_dict[field]:
+                missing_fields.append(field)
+
+        if missing_fields:
+            item_type = "asset" if isinstance(item, IntegrationAsset) else "finding"
+            raise ValueError(f"Missing or empty required fields for {item_type}: {', '.join(missing_fields)}")
+
+    def create_artifacts_dir(self) -> Path:
+        """Create artifacts directory if it doesn't exist."""
+        artifacts_dir = Path("./artifacts")
+        artifacts_dir.mkdir(exist_ok=True, parents=True)
+        return artifacts_dir
+
+    def _get_item_key(self, item_dict: Dict[str, Any], item_type: str) -> str:
+        """Generate a unique key for an item (asset or finding) dictionary."""
+        if item_type == "asset":
+            return item_dict.get("identifier", "unknown")
+        else:  # finding
+            asset_id = item_dict.get("asset_identifier", "unknown")
+            cve = item_dict.get("cve", "")
+            title = item_dict.get("title", "")
+            if cve:
+                return f"{asset_id}:{cve}"
+            return f"{asset_id}:{title}"
+
+    def _prepare_output_file(self, output_file: str, empty_file: bool, item_type: str) -> Dict[str, bool]:
+        """Prepare output file and load existing records if necessary."""
+        existing_items: Dict[str, bool] = {}
+
+        if empty_file and os.path.exists(output_file):
+            logger.info(f"Emptying existing file: {output_file}")
+            open(output_file, "w").close()
+        elif os.path.exists(output_file) and os.path.getsize(output_file) > 0:
+            logger.info(f"Reading existing records from: {output_file}")
+            try:
+                with open(output_file, "r") as f:
+                    for line in f:
+                        try:
+                            record = json.loads(line.strip())
+                            key = self._get_item_key(record, item_type)
+                            existing_items[key] = True
+                        except json.JSONDecodeError:
+                            logger.warning(f"Could not parse line in {output_file}")
+            except Exception as e:
+                logger.warning(f"Error reading existing records: {str(e)}")
+
+        return existing_items
+
+    def _write_items_to_jsonl(
+        self,
+        file_path: str,
+        output_file: str,
+        item_type: str,
+        empty_file: bool = True,
+    ) -> int:
+        """
+        Process source files (local or S3) and write items (assets or findings) to JSONL.
+
+        :param str file_path: Path to source file or directory (local or S3 URI)
+        :param str output_file: Path to output JSONL file
+        :param str item_type: Type of items to process ('asset' or 'finding')
+        :param bool empty_file: Whether to empty the output file before writing (default: True)
+        :return: Total count of items written
+        :rtype: int
+        """
+        existing_items = self._prepare_output_file(output_file, empty_file, item_type)
+        total_items_count = len(existing_items)
+        processed_files = set()
+        new_items_count = 0
+
+        with open(output_file, "a") as output_f:
+            for file_data in self.find_valid_files(file_path):
+                if isinstance(file_data, tuple) and len(file_data) >= 2:
+                    file, data = file_data[0], file_data[1]
+                else:
+                    file, data = file_data, None
+
+                file_str = str(file)
+                if file_str in processed_files:
+                    continue
+
+                processed_files.add(file_str)
+
+                try:
+                    logger.info(f"Processing file: {file}")
+                    if item_type == "asset":
+                        self._process_asset_file(file, data, output_f, existing_items)
+                        new_items_count += 1
+                        total_items_count += 1
+                    else:
+                        findings_count = self._process_finding_file(file, data, output_f, existing_items)
+                        new_items_count += findings_count
+                        total_items_count += findings_count
+
+                except Exception as e:
+                    logger.error(f"Error processing file {file}: {str(e)}")
+
+        item_type_label = "assets" if item_type == "asset" else "findings"
+        logger.info(f"Added {new_items_count} new {item_type_label} to {output_file}")
+        return total_items_count
+
+    def _process_asset_file(self, file, data, output_f, existing_items):
+        """
+        Process a single file for assets with mapping and validation.
+
+        :param file: The file being processed
+        :param data: The data from the file
+        :param output_f: The output file handle
+        :param existing_items: Dictionary of existing items
+        :return: Number of assets processed
+        :rtype: int
+        """
+        asset = self.parse_asset(file, data)
+        asset_dict = dataclasses.asdict(asset)
+
+        if not self.disable_mapping:
+            mapped_asset_dict = self._apply_mapping(
+                data or {},
+                asset_dict,
+                getattr(self.mapping, "fields", {}).get("asset_mapping", {}) if self.mapping else {},
+            )
+            mapped_asset = IntegrationAsset(**mapped_asset_dict)
+        else:
+            mapped_asset = asset
+
+        self._validate_fields(mapped_asset, self.required_asset_fields)
+
+        key = self._get_item_key(dataclasses.asdict(mapped_asset), "asset")
+        if key in existing_items:
+            logger.debug(f"Asset with identifier {key} already exists, skipping")
+            return 0
+
+        output_f.write(json.dumps(dataclasses.asdict(mapped_asset)) + "\n")
+        output_f.flush()
+        existing_items[key] = True
+        return 1
+
+    def _process_finding_file(self, file, data, output_f, existing_items):
+        """
+        Process a single file for findings with mapping and validation.
+
+        :param file: The file being processed
+        :param data: The data from the file
+        :param output_f: The output file handle
+        :param existing_items: Dictionary of existing items
+        :return: Number of findings processed
+        :rtype: int
+        """
+        asset = self.parse_asset(file, data)
+        identifier = asset.identifier
+        findings_data = self._get_findings_data_from_file(data)
+
+        findings_in_file = 0
+        for finding_item in findings_data:
+            finding = self.parse_finding(identifier, data, finding_item)
+            finding_dict = dataclasses.asdict(finding)
+
+            if not self.disable_mapping:
+                mapped_finding_dict = self._apply_mapping(
+                    finding_item,
+                    finding_dict,
+                    getattr(self.mapping, "fields", {}).get("finding_mapping", {}) if self.mapping else {},
+                )
+                mapped_finding = IntegrationFinding(**mapped_finding_dict)
+            else:
+                mapped_finding = finding
+
+            self._validate_fields(mapped_finding, self.required_finding_fields)
+
+            key = self._get_item_key(dataclasses.asdict(mapped_finding), "finding")
+            if key in existing_items:
+                logger.debug(f"Finding with key {key} already exists, skipping")
+                continue
+
+            output_f.write(json.dumps(dataclasses.asdict(mapped_finding)) + "\n")
+            output_f.flush()
+            existing_items[key] = True
+            findings_in_file += 1
+
+        if findings_in_file > 0:
+            logger.info(f"Added {findings_in_file} new findings from file {file}")
+        return findings_in_file
+
+    def _get_findings_data_from_file(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Extract findings data from file data (default implementation).
+
+        Subclasses must override this method to extract findings data from their specific file format.
+
+        :param Dict[str, Any] data: The data from the file
+        :return: Iterable of finding items
+        """
+        # Default implementation returns an empty list
+        # Subclasses must override this method
+        return []
+
+    def _yield_items_from_jsonl(self, jsonl_file: str, item_class: Type[ItemType]) -> Iterator[ItemType]:
+        """
+        Read items from JSONL file and yield them one by one.
+
+        :param str jsonl_file: Path to JSONL file containing items
+        :param Type[ItemType] item_class: Class to convert dictionary items to (IntegrationAsset or IntegrationFinding)
+        :yields: Items one at a time
+        :rtype: Iterator[ItemType]
+        """
+        if not os.path.exists(jsonl_file):
+            logger.warning(f"JSONL file {jsonl_file} does not exist")
+            return
+
+        logger.info(f"Reading items from {jsonl_file}")
+        with open(jsonl_file, "r") as f:
+            for line_number, line in enumerate(f, 1):
+                try:
+                    item_dict = json.loads(line.strip())
+                    yield item_class(**item_dict)
+                except json.JSONDecodeError:
+                    logger.warning(f"Could not parse line {line_number} in {jsonl_file}")
+                except Exception as e:
+                    logger.error(f"Error processing line {line_number} in {jsonl_file}: {str(e)}")
+
+    def _process_files(
+        self,
+        file_path: Union[str, Path],
+        assets_output_file: str,
+        findings_output_file: str,
+        empty_assets_file: bool = True,
+        empty_findings_file: bool = True,
+    ) -> Tuple[int, int]:
+        """
+        Process files (local or S3) to extract both assets and findings in a single pass.
+
+        Optimizes file processing by reading each file once to extract asset and finding data.
+
+        :param Union[str, Path] file_path: Path to source file or directory (local or S3 URI)
+        :param str assets_output_file: Path to output JSONL file for assets
+        :param str findings_output_file: Path to output JSONL file for findings
+        :param bool empty_assets_file: Whether to empty the assets file before writing (default: True)
+        :param bool empty_findings_file: Whether to empty the findings file before writing (default: True)
+        :return: Tuple of total asset and finding counts
+        :rtype: Tuple[int, int]
+        """
+        existing_assets = self._prepare_output_file(assets_output_file, empty_assets_file, "asset")
+        existing_findings = self._prepare_output_file(findings_output_file, empty_findings_file, "finding")
+
+        asset_tracker = self._init_tracker(existing_assets)
+        finding_tracker = self._init_tracker(existing_findings)
+        processed_files = set()
+
+        with open(assets_output_file, "a") as assets_file, open(findings_output_file, "a") as findings_file:
+            for file, data in self._get_valid_file_data(file_path):
+                if str(file) in processed_files:
+                    continue
+
+                processed_files.add(str(file))
+                self._process_file(file, data, assets_file, findings_file, asset_tracker, finding_tracker)
+
+        self._log_processing_results(asset_tracker.new_count, assets_output_file, "assets")
+        self._log_processing_results(finding_tracker.new_count, findings_output_file, "findings")
+        return asset_tracker.total_count, finding_tracker.total_count
+
+    def _init_tracker(self, existing_items: Dict[str, bool]) -> "CountTracker":
+        """
+        Initialize a tracker for counting new and total items.
+
+        :param Dict[str, bool] existing_items: Dictionary of existing item keys
+        :return: Tracker object for managing counts
+        :rtype: CountTracker
+        """
+        from dataclasses import dataclass
+
+        @dataclass
+        class CountTracker:
+            existing: Dict[str, bool]
+            new_count: int = 0
+            total_count: int = 0
+
+        return CountTracker(existing=existing_items, total_count=len(existing_items))
+
+    def _get_valid_file_data(
+        self, file_path: Union[str, Path]
+    ) -> Iterator[Tuple[Union[Path, str], Optional[Dict[str, Any]]]]:
+        """
+        Yield valid file data from the given path.
+
+        :param Union[str, Path] file_path: Path to source file or directory (local or S3 URI)
+        :return: Iterator yielding tuples of (file path, parsed data)
+        :rtype: Iterator[Tuple[Union[Path, str], Optional[Dict[str, Any]]]]
+        """
+        for file_data in self.find_valid_files(file_path):
+            if isinstance(file_data, tuple) and len(file_data) >= 2:
+                yield file_data[0], file_data[1]
+            else:
+                yield file_data, None
+
+    def _process_file(
+        self,
+        file: Union[Path, str],
+        data: Optional[Dict[str, Any]],
+        assets_file: Any,
+        findings_file: Any,
+        asset_tracker: "CountTracker",
+        finding_tracker: "CountTracker",
+    ) -> None:
+        """
+        Process a single file for assets and findings.
+
+        :param Union[Path, str] file: Path to the file being processed
+        :param Optional[Dict[str, Any]] data: Parsed data from the file
+        :param Any assets_file: Open file handle for writing assets
+        :param Any findings_file: Open file handle for writing findings
+        :param CountTracker asset_tracker: Tracker for asset counts
+        :param CountTracker finding_tracker: Tracker for finding counts
+        :rtype: None
+        """
+        try:
+            logger.info(f"Processing file: {file}")
+            self._process_asset(file, data, assets_file, asset_tracker)
+            self._process_findings(file, data, findings_file, asset_tracker.existing, finding_tracker)
+        except Exception as e:
+            logger.error(f"Error processing file {file}: {str(e)}")
+
+    def _process_asset(
+        self,
+        file: Union[Path, str],
+        data: Optional[Dict[str, Any]],
+        assets_file: Any,
+        tracker: "CountTracker",
+    ) -> None:
+        """
+        Process and write a single asset from file data.
+
+        :param Union[Path, str] file: Path to the file being processed
+        :param Optional[Dict[str, Any]] data: Parsed data from the file
+        :param Any assets_file: Open file handle for writing assets
+        :param CountTracker tracker: Tracker for asset counts
+        :rtype: None
+        """
+        asset = self.parse_asset(file, data)
+        asset_dict = dataclasses.asdict(asset)
+        mapped_asset = self._map_item(asset_dict, "asset_mapping", IntegrationAsset)
+        self._validate_fields(mapped_asset, self.required_asset_fields)
+
+        asset_key = mapped_asset.identifier
+        if asset_key not in tracker.existing:
+            self._write_item(assets_file, mapped_asset)
+            tracker.existing[asset_key] = True
+            tracker.new_count += 1
+            tracker.total_count += 1
+        else:
+            logger.debug(f"Asset with identifier {asset_key} already exists, skipping")
+
+    def _process_findings(
+        self,
+        file: Union[Path, str],
+        data: Optional[Dict[str, Any]],
+        findings_file: Any,
+        existing_assets: Dict[str, bool],
+        tracker: "CountTracker",
+    ) -> None:
+        """
+        Process and write findings from file data.
+
+        :param Union[Path, str] file: Path to the file being processed
+        :param Optional[Dict[str, Any]] data: Parsed data from the file
+        :param Any findings_file: Open file handle for writing findings
+        :param Dict[str, bool] existing_assets: Dictionary of existing asset keys
+        :param CountTracker tracker: Tracker for finding counts
+        :rtype: None
+        """
+        findings_data = self._get_findings_data_from_file(data)
+        logger.info(f"Found {len(findings_data)} findings in file: {file}")
+        findings_added = 0
+
+        asset_id = list(existing_assets.keys())[0] if existing_assets else "unknown"
+        for finding_item in findings_data:
+            finding = self.parse_finding(asset_id, data, finding_item)
+            finding_dict = dataclasses.asdict(finding)
+            mapped_finding = self._map_item(finding_dict, "finding_mapping", IntegrationFinding)
+            self._validate_fields(mapped_finding, self.required_finding_fields)
+
+            finding_key = self._get_item_key(dataclasses.asdict(mapped_finding), "finding")
+            if finding_key not in tracker.existing:
+                self._write_item(findings_file, mapped_finding)
+                tracker.existing[finding_key] = True
+                tracker.new_count += 1
+                tracker.total_count += 1
+                findings_added += 1
+            else:
+                logger.debug(f"Finding with key {finding_key} already exists, skipping")
+
+        if findings_added > 0:
+            logger.info(f"Added {findings_added} new findings from file {file}")
+
+    def _map_item(self, item_dict: Dict[str, Any], mapping_key: str, item_class: Type) -> Any:
+        """
+        Apply mapping to an item dictionary if enabled.
+
+        :param Dict[str, Any] item_dict: Dictionary of item data
+        :param str mapping_key: Key in the mapping configuration to use (e.g., 'asset_mapping')
+        :param Type item_class: Class to instantiate with mapped data (IntegrationAsset or IntegrationFinding)
+        :return: Instantiated item object with mapped data
+        :rtype: Any
+        """
+        if not self.disable_mapping and self.mapping and hasattr(self.mapping, "fields"):
+            mapped_dict = self._apply_mapping(
+                item_dict, item_dict, getattr(self.mapping, "fields", {}).get(mapping_key, {})
+            )
+            return item_class(**mapped_dict)
+        return item_class(**item_dict)
+
+    def _write_item(self, file_handle: Any, item: Any) -> None:
+        """
+        Write an item to the specified file handle.
+
+        :param Any file_handle: Open file handle to write to
+        :param Any item: Item object to write (IntegrationAsset or IntegrationFinding)
+        :rtype: None
+        """
+        file_handle.write(json.dumps(dataclasses.asdict(item)) + "\n")
+        file_handle.flush()
+
+    def _log_processing_results(self, new_count: int, output_file: str, item_type: str) -> None:
+        """
+        Log the results of processing items.
+
+        :param int new_count: Number of new items added
+        :param str output_file: Path to the output file
+        :param str item_type: Type of items processed ('assets' or 'findings')
+        :rtype: None
+        """
+        logger.info(f"Added {new_count} new {item_type} to {output_file}")
+
+    def _validate_file_path(self, file_path: Optional[str]) -> str:
+        """
+        Validates the file path and raises an exception if it's invalid.
+
+        :param Optional[str] file_path: Path to validate
+        :return: The validated file path
+        :rtype: str
+        :raises ValidationException: If the file path is invalid
+        """
+        if not file_path:
+            logger.error("No file path provided")
+            raise ValidationException("File path is required")
+
+        if not is_s3_path(file_path) and not os.path.exists(file_path):
+            logger.error(f"File path does not exist: {file_path}")
+            raise ValidationException(f"Path does not exist: {file_path}")
+
+        return file_path
+
+    def fetch_assets(self, *args: Any, **kwargs: Any) -> Iterator[IntegrationAsset]:
+        """
+        Fetches assets from processed source files (local or S3).
+
+        This method supports both local files/directories and S3 paths.
+
+        :param str file_path: Path to a source file or directory
+        :param bool empty_file: Whether to empty the output file before writing (default: True)
+        :param bool process_together: Whether to process assets and findings together (default: False)
+        :param bool use_jsonl_file: Whether to use an existing JSONL file instead of processing source files
+        (default: False)
+        :yields: Iterator[IntegrationAsset]
+        """
+        logger.info("Starting fetch_assets")
+        file_path = kwargs.get("file_path", self.file_path)
+        empty_file = kwargs.get("empty_file", True)
+        process_together = kwargs.get("process_together", False)
+        use_jsonl_file = kwargs.get("use_jsonl_file", False)
+
+        self.create_artifacts_dir()
+
+        if use_jsonl_file:
+            logger.info(f"Using existing JSONL file: {self.ASSETS_FILE}")
+            total_assets = sum(1 for _ in open(self.ASSETS_FILE, "r")) if os.path.exists(self.ASSETS_FILE) else 0
+            self.num_assets_to_process = total_assets
+            logger.info(f"Found {total_assets} assets in existing JSONL file")
+        else:
+            file_path = self._validate_file_path(file_path)
+            if process_together:
+                total_assets, _ = self._process_files(
+                    file_path,
+                    self.ASSETS_FILE,
+                    self.FINDINGS_FILE,
+                    empty_assets_file=empty_file,
+                    empty_findings_file=False,
+                )
+                self.num_assets_to_process = total_assets
+            else:
+                total_assets = self._write_items_to_jsonl(file_path, self.ASSETS_FILE, "asset", empty_file=empty_file)
+                self.num_assets_to_process = total_assets
+            logger.info(f"Total assets to process: {total_assets}")
+
+        for asset in self._yield_items_from_jsonl(self.ASSETS_FILE, IntegrationAsset):
+            yield asset
+
+        logger.info(f"Assets read from JSONL complete. Total assets identified: {self.num_assets_to_process}")
+
+    def fetch_findings(self, *args: Any, **kwargs: Any) -> Iterator[IntegrationFinding]:
+        """
+        Fetches findings from processed source files (local or S3).
+
+        This method supports both local files/directories and S3 paths.
+
+        :param str file_path: Path to source file or directory
+        :param bool empty_file: Whether to empty the output file before writing (default: True)
+        :param bool process_together: Whether to process assets and findings together (default: False)
+        :param bool use_jsonl_file: Whether to use an existing JSONL file instead of processing source files (default: False)
+        :yields: Iterator[IntegrationFinding]
+        """
+        logger.info("Starting fetch_findings")
+        file_path = kwargs.get("file_path", self.file_path)
+        empty_file = kwargs.get("empty_file", True)
+        process_together = kwargs.get("process_together", False)
+        use_jsonl_file = kwargs.get("use_jsonl_file", False)
+
+        self.create_artifacts_dir()
+
+        if use_jsonl_file:
+            logger.info(f"Using existing JSONL file: {self.FINDINGS_FILE}")
+            total_findings = sum(1 for _ in open(self.FINDINGS_FILE, "r")) if os.path.exists(self.FINDINGS_FILE) else 0
+            self.num_findings_to_process = total_findings
+            logger.info(f"Found {total_findings} findings in existing JSONL file")
+        else:
+            file_path = self._validate_file_path(file_path)
+            if process_together:
+                _, total_findings = self._process_files(
+                    file_path,
+                    self.ASSETS_FILE,
+                    self.FINDINGS_FILE,
+                    empty_assets_file=False,
+                    empty_findings_file=empty_file,
+                )
+                self.num_findings_to_process = total_findings
+            else:
+                total_findings = self._write_items_to_jsonl(
+                    file_path, self.FINDINGS_FILE, "finding", empty_file=empty_file
+                )
+                self.num_findings_to_process = total_findings
+            logger.info(f"Total findings to process: {total_findings}")
+
+        for finding in self._yield_items_from_jsonl(self.FINDINGS_FILE, IntegrationFinding):
+            yield finding
+
+        logger.info(f"Findings read from JSONL complete. Total findings identified: {self.num_findings_to_process}")
+
+    def fetch_assets_and_findings(
+        self, file_path: str = None, empty_files: bool = True
+    ) -> Tuple[Iterator[IntegrationAsset], Iterator[IntegrationFinding]]:
+        """Process both assets and findings (local or S3) in a single pass and return iterators.
+
+        This method optimizes the processing by reading each file only once and extracting
+        both asset and finding information in a single pass. It returns two iterators,
+        one for assets and one for findings.
+
+        :param str file_path: Path to source file or directory
+        :param bool empty_files: Whether to empty both output files before writing (default: True)
+        :return: Tuple of (assets_iterator, findings_iterator)
+        :rtype: Tuple[Iterator[IntegrationAsset], Iterator[IntegrationFinding]]
+        """
+        file_path = self._validate_file_path(file_path or self.file_path)
+        self.create_artifacts_dir()
+
+        logger.info("Processing assets and findings together from %s", file_path)
+        total_assets, total_findings = self._process_files(
+            file_path=file_path,
+            assets_output_file=self.ASSETS_FILE,
+            findings_output_file=self.FINDINGS_FILE,
+            empty_assets_file=empty_files,
+            empty_findings_file=empty_files,
+        )
+
+        self.num_assets_to_process = total_assets
+        self.num_findings_to_process = total_findings
+
+        assets_iterator = self._yield_items_from_jsonl(self.ASSETS_FILE, IntegrationAsset)
+        findings_iterator = self._yield_items_from_jsonl(self.FINDINGS_FILE, IntegrationFinding)
+        return assets_iterator, findings_iterator
+
+    def sync_assets_and_findings(self) -> None:
+        """Process both assets and findings (local or S3) in a single pass and sync to RegScale.
+
+        This method optimizes the processing by reading each file only once and
+        extracting both asset and finding information in a single pass.
+
+        :param int plan_id: RegScale Security Plan ID
+        :param str file_path: Path to source file or directory
+        :param bool empty_files: Whether to empty both output files before writing (default: True)
+        :rtype: None
+        """
+        file_path = self._validate_file_path(self.file_path)
+        logger.info("Processing assets and findings together from %s", file_path)
+        total_assets, total_findings = self._process_files(
+            file_path=file_path,
+            assets_output_file=self.ASSETS_FILE,
+            findings_output_file=self.FINDINGS_FILE,
+            empty_assets_file=self.empty_files,
+            empty_findings_file=self.empty_files,
+        )
+
+        logger.info("Syncing %d assets to RegScale", total_assets)
+        self.sync_assets(
+            plan_id=self.plan_id,
+            file_path=file_path,
+            use_jsonl_file=True,
+            asset_count=total_assets,
+        )
+
+        logger.info("Syncing %d findings to RegScale", total_findings)
+        self.sync_findings(
+            plan_id=self.plan_id,
+            file_path=file_path,
+            use_jsonl_file=True,
+            finding_count=total_findings,
+        )
+
+        logger.info("Assets and findings sync complete")
+
+    # Abstract method with default implementation for reading files
+    def find_valid_files(self, path: Union[Path, str]) -> Iterator[Tuple[Union[Path, str], Dict[str, Any]]]:
+        """
+        Find all valid source files in the given path and read their contents if read_files_only is True.
+
+        Subclasses must override this method to customize file validation and data extraction.
+
+        :param Union[Path, str] path: Path to a file or directory (local or S3 URI)
+        :return: Iterator yielding tuples of (file path, validated data)
+        :rtype: Iterator[Tuple[Union[Path, str], Dict[str, Any]]]
+        """
+        for file in find_files(path, self.file_pattern):
+            data = self._read_file_content(file)
+            if data is not None:
+                yield from self._validate_and_yield(file, data)
+
+    def _read_file_content(self, file: Union[Path, str]) -> Optional[Dict[str, Any]]:
+        """
+        Read and parse the content of a file based on read_files_only setting.
+
+        :param Union[Path, str] file: Path to the file to read
+        :return: Parsed JSON data or None if reading fails
+        :rtype: Optional[Dict[str, Any]]
+        """
+        try:
+            if self.read_files_only:
+                return self._read_content_directly(file)
+            return self._read_content_with_download(file)
+        except json.JSONDecodeError:
+            logger.warning(f"File {file} is not valid JSON, skipping")
+            return None
+        except Exception as e:
+            logger.error(f"Error reading file {file}: {str(e)}")
+            return None
+
+    def _read_content_directly(self, file: Union[Path, str]) -> Dict[str, Any]:
+        """
+        Read file content directly when read_files_only is True.
+
+        :param Union[Path, str] file: Path to the file
+        :return: Parsed JSON data
+        :rtype: Dict[str, Any]
+        """
+        content = read_file(file)
+        return json.loads(content) if content else {}
+
+    def _read_content_with_download(self, file: Union[Path, str]) -> Dict[str, Any]:
+        """
+        Read file content, downloading from S3 if necessary, when read_files_only is False.
+
+        :param Union[Path, str] file: Path to the file (local or S3 URI)
+        :return: Parsed JSON data
+        :rtype: Dict[str, Any]
+        """
+        if is_s3_path(file):
+            temp_dir = Path(tempfile.mkdtemp())
+            try:
+                s3_parts = file[5:].split("/", 1)
+                bucket = s3_parts[0]
+                prefix = s3_parts[1] if len(s3_parts) > 1 else ""
+                download_from_s3(bucket, prefix, temp_dir, self.aws_profile)
+                local_file = temp_dir / os.path.basename(prefix)
+                with open(local_file, "r") as f:
+                    return json.load(f)
+            finally:
+                shutil.rmtree(temp_dir)
+        else:
+            with open(file, "r") as f:
+                return json.load(f)
+
+    def _validate_and_yield(
+        self, file: Union[Path, str], data: Dict[str, Any]
+    ) -> Iterator[Tuple[Union[Path, str], Dict[str, Any]]]:
+        """
+        Validate file data and yield it if valid.
+
+        :param Union[Path, str] file: Path to the file
+        :param Dict[str, Any] data: Parsed data from the file
+        :return: Iterator yielding valid file data tuples
+        :rtype: Iterator[Tuple[Union[Path, str], Dict[str, Any]]]
+        """
+        is_valid, validated_data = self.is_valid_file(data, file)
+        if is_valid and validated_data is not None:
+            yield file, validated_data
+
+    def parse_asset(self, file_path: Union[Path, str], data: Dict[str, Any]) -> IntegrationAsset:
+        """
+        Parse a single asset from source data.
+
+        Subclasses must implement this method to parse assets from their specific file format.
+
+        :param Union[Path, str] file_path: Path to the file containing the asset data
+        :param Dict[str, Any] data: The parsed data
+        :return: IntegrationAsset object
+        :rtype: IntegrationAsset
+        """
+        raise NotImplementedError("Subclasses must implement parse_asset")
+
+    def parse_finding(self, asset_identifier: str, data: Dict[str, Any], item: Dict[str, Any]) -> IntegrationFinding:
+        """Parse a single finding from source data.
+
+        Subclasses must implement this method to parse findings from their specific file format.
+
+        :param str asset_identifier: The identifier of the asset this finding belongs to
+        :param Dict[str, Any] data: The asset data
+        :param Dict[str, Any] item: The finding data
+        :return: IntegrationFinding object
+        :rtype: IntegrationFinding
+        """
+        raise NotImplementedError("Subclasses must implement parse_finding")
+
+    def is_valid_file(self, data: Any, file_path: Union[Path, str]) -> Tuple[bool, Optional[Dict[str, Any]]]:
+        """
+        Check if the provided data is valid for processing.
+
+        This default implementation ensures the data is a non-empty dictionary.
+        Subclasses should override this to implement specific validation logic.
+
+        :param Any data: Data parsed from the file to validate
+        :param Union[Path, str] file_path: Path to the file being processed
+        :return: Tuple of (is_valid, data) where is_valid indicates validity and data is the validated content or None
+        :rtype: Tuple[bool, Optional[Dict[str, Any]]]
+        """
+        if not isinstance(data, dict):
+            logger.warning(f"Data is not a dictionary for file {file_path}, skipping")
+            return False, None
+
+        if not data:
+            logger.warning(f"Data is an empty dictionary for file {file_path}, skipping")
+            return False, None
+
+        return True, data