stix-shifter-modules-sysdig 8.0.2__py3-none-any.whl

stix_shifter_modules/sysdig/stix_transmission/connector.py

@@ -0,0 +1,213 @@
+import json
+from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector
+from stix_shifter_utils.utils.error_response import ErrorResponder
+from stix_shifter_utils.utils import logger
+from .api_client import APIClient
+
+
+class InvalidMetadataException(Exception):
+    pass
+
+
+class Connector(BaseJsonSyncConnector):
+    SYSDIG_MAX_PAGE_SIZE = 999
+
+    def __init__(self, connection, configuration):
+        self.api_client = APIClient(connection, configuration)
+        self.logger = logger.set_logger(__name__)
+        self.connector = __name__.split('.')[1]
+
+    async def create_results_connection(self, query, offset, length, metadata=None):
+        """
+        Fetching the results using query, offset and length
+        :param query: str, Data Source query
+        :param offset: str, Offset value
+        :param length: str, Length value
+        :param metadata: dict
+        :return: return_obj, dict
+        """
+        return_obj = {}
+        response_dict = {}
+        data = []
+        local_result_count = 0
+        # Using 'prev' cursor instead of 'next' cursor in Sysdig since the records are returned in desc order by last
+        # modified timestamp. The 'prev' is not None when all the results are returned in the first api call itself
+        # and there are no more records.
+        # Hence, checking for result_count returned from API call with the limit. When the result_count is less than
+        # the limit, setting prev_page_token to None. This is to overcome Sysdig behaviour.
+        try:
+            if metadata:
+                if isinstance(metadata, dict) and metadata.get('result_count') and metadata.get('prev_page_token'):
+                    result_count, prev_page_token = metadata['result_count'], metadata['prev_page_token']
+                    result_count = int(result_count)
+                    total_records = int(length)
+                    if abs(self.api_client.result_limit - result_count) < total_records:
+                        total_records = abs(self.api_client.result_limit - result_count)
+
+                else:
+                    # Raise exception when metadata doesnt contain result count or next page token.
+                    raise InvalidMetadataException(metadata)
+            else:
+                result_count, prev_page_token = 0, '0'
+                total_records = int(offset) + int(length)
+                if total_records > int(self.api_client.result_limit):
+                    total_records = int(self.api_client.result_limit)
+            if total_records <= Connector.SYSDIG_MAX_PAGE_SIZE:
+                limit = total_records
+            else:
+                limit = Connector.SYSDIG_MAX_PAGE_SIZE
+            query_limit = limit
+            limit = f'&limit={limit}'
+            if (result_count == 0 and prev_page_token == '0') or (prev_page_token != '0' and result_count <
+                                                                  self.api_client.result_limit):
+                """api call for searching alert based on query"""
+                split_query = query.split('&filter=')
+                if metadata:
+                    response_wrapper = await self.api_client.get_search_results(
+                        f'&cursor={prev_page_token}&filter={split_query[1]}{limit}')
+                else:
+                    response_wrapper = await self.api_client.get_search_results(query + limit)
+                response_code = response_wrapper.code
+                if response_code == 200:
+                    response_dict = json.loads(response_wrapper.read())
+                    return_obj['success'] = True
+                    result_count += len(response_dict['data'])
+                    local_result_count += len(response_dict['data'])
+                    prev_page_token = response_dict['page'].get('prev')
+                    if len(response_dict['data']) < query_limit:
+                        prev_page_token = None
+                    data += response_dict['data']
+                    # Loop until if there is next page and records fetched is less than total records.
+                    while prev_page_token:
+                        if not metadata and result_count < total_records:
+                            remaining_records = total_records - result_count
+                        elif metadata and local_result_count < total_records:
+                            remaining_records = total_records - local_result_count
+                        else:
+                            break
+                        if remaining_records > Connector.SYSDIG_MAX_PAGE_SIZE:
+                            limit = Connector.SYSDIG_MAX_PAGE_SIZE
+                        else:
+                            limit = remaining_records
+                        query_limit = limit
+                        limit = f'&limit={limit}'
+                        prev_response_wrapper = await self.api_client.get_search_results(
+                            f'&cursor={prev_page_token}&filter={split_query[1]}{limit}')
+                        prev_response_code = prev_response_wrapper.code
+                        prev_response_dict = json.loads(prev_response_wrapper.read())
+                        if prev_response_code == 200:
+                            result_count += len(prev_response_dict['data'])
+                            local_result_count += len(response_dict['data'])
+                            prev_page_token = prev_response_dict['page'].get('prev')
+                            if len(prev_response_dict['data']) < query_limit:
+                                prev_page_token = None
+                            data += prev_response_dict['data']
+                        else:
+                            return_obj = self.exception_response(prev_response_wrapper.code,
+                                                                 prev_response_dict.get('error'))
+                            data = []
+                            break
+                    if data:
+                        self.update_event(data)
+
+                        if metadata:
+                            return_obj['data'] = data if data else []
+                        else:
+                            return_obj['data'] = data[int(offset):total_records] if data else []
+
+                        if prev_page_token and result_count < self.api_client.result_limit:
+                            return_obj['metadata'] = {"result_count": result_count,
+                                                      "prev_page_token": str(prev_page_token)}
+                    else:
+                        if not return_obj.get('error') and return_obj.get('success') is not False:
+                            return_obj['success'] = True
+                            return_obj['data'] = []
+                else:
+                    if response_code == 401:
+                        message = 'cannot verify credentials'
+                    else:
+                        response_dict = json.loads(response_wrapper.read())
+                        message = response_dict.get('message')
+                        if response_code == 400 and message == 'bad request':
+                            message = 'Query time range can be maximum up to 14 days or The given query is invalid'
+                    return_obj = self.exception_response(response_wrapper.code, message)
+            else:
+                return_obj['success'] = True
+                return_obj['data'] = []
+
+        except InvalidMetadataException as ex:
+            response_dict['code'] = 100
+            response_dict['message'] = f'Invalid metadata: {str(ex)}'
+            ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector)
+
+        except Exception as ex:
+            if "timeout_error" in str(ex):
+                response_dict['code'] = 408
+            response_dict['message'] = str(ex)
+            self.logger.error(f'{self.connector} connector error when getting search results: %s', ex)
+            ErrorResponder.fill_error(return_obj, response_dict, ['message'],
+                                      connector=self.connector)
+        return return_obj
+
+    @staticmethod
+    def update_event(data):
+        """
+        Set the finding_type as 'threat' for the event.
+        Replace invalid <NA> tag with empty string.
+        Parse the fd.name and set the fields for network-traffic in the event.
+        """
+        for event in data:
+            fields = event["content"]["fields"]
+            # replace unsupported field value <NA> with empty string
+            for f in fields:
+                if fields[f] == '<NA>':
+                    fields[f] = ''
+
+            # Setting the finding_type for the event as 'threat'
+            event["finding_type"] = "threat"
+            # Parsing network fields from event data
+            if fields.get("evt.type") and fields["evt.type"] == "connect":
+                if "->" in fields["fd.name"]:
+                    source, destination = fields["fd.name"].split('->')
+                    event["clientIpv4"], event["clientPort"] = source.split(":")
+                    event["serverIpv4"], event["serverPort"] = destination.split(":")
+                if fields.get('fd.l4proto'):
+                    event["l4protocol"] = fields['fd.l4proto']
+                else:
+                    event["l4protocol"] = "tcp"
+
+            # Adding ancestor names in list
+            fields["proc.anames"] = [val for key, val in fields.items() if 'proc.aname' in key and len(val) > 0]
+
+    async def ping_connection(self):
+        """
+        Ping the endpoint
+        :return: dict
+        """
+        return_obj = {}
+        response_dict = {}
+        try:
+            response = await self.api_client.ping_data_source()
+            response_code = response.code
+            if response_code == 200:
+                return_obj['success'] = True
+            else:
+                return_obj = self.exception_response(response_code, 'cannot verify credentials')
+        except Exception as ex:
+            response_dict['message'] = ex
+            self.logger.error(f'{self.connector} connector error while pinging: %s', ex)
+            ErrorResponder.fill_error(return_obj, response_dict, ['message'],
+                                      connector=self.connector)
+        return return_obj
+
+    def exception_response(self, code, response_txt):
+        """
+        create the exception response
+        :param code, int
+        :param response_txt, dict
+        :return: return_obj, dict
+        """
+        return_obj = {}
+        response_dict = {'code': code, 'message': str(response_txt)}
+        ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector)
+        return return_obj

stix_shifter_modules/sysdig/stix_transmission/error_mapper.py

@@ -0,0 +1,34 @@
+from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase
+from stix_shifter_utils.utils.error_response import ErrorCode
+from stix_shifter_utils.utils import logger
+
+error_mapping = {
+    400: ErrorCode.TRANSMISSION_INVALID_PARAMETER,
+    422: ErrorCode.TRANSMISSION_QUERY_LOGICAL_ERROR,
+    100: ErrorCode.TRANSMISSION_INVALID_PARAMETER,
+    401: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS,
+    408: ErrorCode.TRANSMISSION_CONNECT
+}
+
+
+class ErrorMapper:
+    logger = logger.set_logger(__name__)
+    DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR
+
+    @staticmethod
+    def set_error_code(json_data, return_obj, connector=None):
+        code = None
+        try:
+            code = int(json_data['code'])
+        except Exception:
+            pass
+
+        error_code = ErrorMapper.DEFAULT_ERROR
+
+        if code in error_mapping:
+            error_code = error_mapping[code]
+
+        if error_code == ErrorMapper.DEFAULT_ERROR:
+            ErrorMapper.logger.error("failed to map: " + str(json_data))
+
+        ErrorMapperBase.set_error_code(return_obj, error_code, connector=connector)

stix_shifter_modules_sysdig-8.0.2.dist-info/METADATA

@@ -0,0 +1,148 @@
+Metadata-Version: 2.4
+Name: stix_shifter_modules_sysdig
+Version: 8.0.2
+Summary: Tools and interface to translate STIX formatted results and queries to different data source formats and to set up appropriate connection strings for invoking and triggering actions in openwhisk
+Author: ibm
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/opencybersecurityalliance/stix-shifter
+Project-URL: Source, https://github.com/opencybersecurityalliance/stix-shifter
+Keywords: datasource,stix,translate,transform,transmit
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+License-File: AUTHORS.md
+License-File: NOTICE
+Dynamic: license-file
+
+[![example workflow](https://github.com/opencybersecurityalliance/stix-shifter/actions/workflows/main.yml/badge.svg)](https://github.com/opencybersecurityalliance/stix-shifter/actions)
+[![codecov](https://codecov.io/gh/opencybersecurityalliance/stix-shifter/branch/develop/graph/badge.svg?token=gQvl14peRj)](https://codecov.io/gh/opencybersecurityalliance/stix-shifter)
+
+# Introduction
+
+STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations.
+
+This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more.
+
+In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same.
+
+***Project Documentation***
+
+For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Documentation](https://stix-shifter.readthedocs.io/)
+
+## Installation
+
+The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages:
+
+1. Main stix-shifter package:  `pip install stix-shifter`
+
+2. Stix-shifter Utility package:  `pip install stix-shifter-utils`
+
+3. Desired stix-shifter connector module package:  `pip install stix-shifter-modules-<module name> `
+   Example:  `pip install stix-shifter-modules-qradar`
+
+### Dependencies
+
+STIX-shifter requries Python 3.10 or greater. See the [requirements file](../stix_shifter/requirements.txt) for library dependencies. 
+
+## Usage
+
+STIX-Shifter can use used the following ways:
+
+### As a command line utility
+
+The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. 
+
+More details of the command line option can be found [here](OVERVIEW.md#how-to-use)
+
+```
+$ stix-shifter translate <MODULE NAME> query "<STIX IDENTITY OBJECT>" "<STIX PATTERN>" "<OPTIONS>"
+```
+Example:
+```
+$ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {}
+```
+
+In order to build `stix-shifter` packages from source follow the below prerequisite steps:
+   1. Go to the stix-shifter parent directory
+   2. Optionally, you can create a Python 3 virtual environemnt:
+       `virtualenv -p python3 virtualenv && source virtualenv/bin/activate`
+   3. Run setup: `python -m build_tools.run_build install`
+
+
+### Running from the source
+
+You may also use the `python3 main.py` script. All the options are the same as the command line utility described above.
+
+Example:
+
+```
+python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {}
+```
+
+In order to run `python3 main.py` from the source follow the below prerequisite steps:
+   1. Go to the stix-shifter parent directory
+   2. Optionally, you can create a Python 3 virtual environemnt:
+       `virtualenv -p python3 virtualenv && source virtualenv/bin/activate`
+   3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 -m build_tools.run_build install`. 
+
+**Note:** `build_tools.run_build` only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 -m build_tools.pre_build && pip install -r requirements.txt`
+
+### As a library
+
+You can also use this library to integrate STIX Shifter into your own tools. You can translate a STIX Pattern:
+
+```
+from stix_shifter.stix_translation import stix_translation
+
+translation = stix_translation.StixTranslation()
+response = translation.translate('<MODULE NAME>', 'query', '{}', '<STIX PATTERN>', '<OPTIONS>')
+
+print(response)
+```
+### Use of custom mappings
+
+If a connector has been installed using pip, the process for editing the STIX mappings is different than if you have pulled-down the project. When working locally, you can edit the mapping files directly. See the [mapping files for the MySQL connector](https://github.com/opencybersecurityalliance/stix-shifter/tree/develop/stix_shifter_modules/mysql/stix_translation/json) as an example. Editing the mapping files won't work if the connector has been installed with pip; the setup script of the stix-shifter package includes the mapppings inside `config.json`. This allows stix-shifter to injest custom mappings as part of the connector's configuration.
+
+Refer to [Use of custom mappings](adapter-guide/custom_mappings.md) for more details on how to edit the mappings in the configuration.
+
+# Contributing
+
+We are thrilled you are considering contributing! We welcome all contributors.
+Please read our [guidelines for contributing](CONTRIBUTING.md).
+
+## [Connector Developer Guide](adapter-guide/develop-stix-adapter.md)
+
+
+## [CLI tools and Connector Development Labs](lab/README.md)
+
+
+# Licensing
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+# More Resources
+
+## Join us on Slack!
+
+[Click here](https://docs.google.com/forms/d/1vEAqg9SKBF3UMtmbJJ9qqLarrXN5zeVG3_obedA3DKs) and fill out the form to receive an invite to the Open Cybersecurity Alliance slack instance, then join the #stix-shifter channel, to meet and discuss usage with the team.
+
+## Introduction Webinar!
+
+[Click here](https://ibm.biz/BdzTyA) to view an introduction webinar on STIX Shifter and the use cases it solves for.
+
+## Changelog
+
+- [Changelog](../CHANGELOG.md)

stix_shifter_modules_sysdig-8.0.2.dist-info/RECORD

@@ -0,0 +1,27 @@
+stix_shifter_modules/sysdig/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+stix_shifter_modules/sysdig/entry_point.py,sha256=DlhtiZzEzBCFQj9SV7zDiOMHvyl7lLAiHpcQPPTW85k,424
+stix_shifter_modules/sysdig/configuration/config.json,sha256=BYcxpCkSUR_CG5ezvOCPd7S4Sx24x75hYrdX_GHTBFQ,26696
+stix_shifter_modules/sysdig/configuration/dialects.json,sha256=tewYuoIE-WBBxsLTlOstMkMSUs6sFlEyW1dveC61xwk,78
+stix_shifter_modules/sysdig/configuration/lang_en.json,sha256=C7rcwaqd1ADbXWwemIy0FH4mKTQmr9bJESfaIDF06dI,3071
+stix_shifter_modules/sysdig/stix_translation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+stix_shifter_modules/sysdig/stix_translation/json_to_stix_translator.py,sha256=JQYc8AoRarJU4PlETIBsx_1f2J_axc1YBjiAaDUCokw,26842
+stix_shifter_modules/sysdig/stix_translation/query_constructor.py,sha256=dSmx_y5xw5r4d7O5WGqw2o2WXfI8GDPaSZ2uaBCUfbw,20230
+stix_shifter_modules/sysdig/stix_translation/query_translator.py,sha256=1IsBlz_YYM9s0myfyed9Wy0fqkV82k77UXd_fTEE38w,1029
+stix_shifter_modules/sysdig/stix_translation/transformers.py,sha256=1s4QpIJZZDOHL4DA0o8RibEYazS8lqxXAXkYFisoyR4,2653
+stix_shifter_modules/sysdig/stix_translation/json/config_map.json,sha256=XlUxXoQzZrecmYuLIDeLGWut-5TPRZ4khWBpdSRevlo,698
+stix_shifter_modules/sysdig/stix_translation/json/from_stix_map.json,sha256=JJ72e-ePAcQJFwf-GEizEr8ksC8vUa73DkRoqQYKYmo,2046
+stix_shifter_modules/sysdig/stix_translation/json/operators.json,sha256=ysKmSc3Ha-AZgFTDih_XaFhM4ejET1gKBnapKmcVHe4,465
+stix_shifter_modules/sysdig/stix_translation/json/to_stix_map.json,sha256=ydLkwcmZEPBsjY_ooJxhN5tfknvgWZ7SgRcDwlrNJ3U,7706
+stix_shifter_modules/sysdig/stix_translation/json/stix_2_1/from_stix_map.json,sha256=axdMRopIMSWx8_FuzN19cYlNlfXObWl4-i57xcbJfPE,2048
+stix_shifter_modules/sysdig/stix_translation/json/stix_2_1/to_stix_map.json,sha256=LPQ0Jw0pLcZ7DAmc2nPf2Az49o67B-BKR7zvAjkg7xM,7708
+stix_shifter_modules/sysdig/stix_transmission/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+stix_shifter_modules/sysdig/stix_transmission/api_client.py,sha256=h4t5wFwDleLoKG_ThNw2pIOcJZ_6_VYUsWUBw8DReFE,1654
+stix_shifter_modules/sysdig/stix_transmission/connector.py,sha256=rQ7945EsGsSoqvQPU3klW3zZ49e_Yfl2Wcv8gb__YLU,10774
+stix_shifter_modules/sysdig/stix_transmission/error_mapper.py,sha256=TF8RD8W9idIxMP_uCyi-I7doPodqvrDhknlg0bfz8Wc,1108
+stix_shifter_modules_sysdig-8.0.2.dist-info/licenses/AUTHORS.md,sha256=2o6cakCKaGsd6zHFZHIva-nm93URvbR6HB0ixeZDTG8,996
+stix_shifter_modules_sysdig-8.0.2.dist-info/licenses/LICENSE.md,sha256=ssGEKMVW69oFTBblVhD1YPolqCOyOCJi4wRDRI7xCnU,12786
+stix_shifter_modules_sysdig-8.0.2.dist-info/licenses/NOTICE,sha256=wx-gWE9vSnLJ7YQkrGlMp9VNXOXG57TTxDDRd9CEdYg,1418
+stix_shifter_modules_sysdig-8.0.2.dist-info/METADATA,sha256=6fyIOoVlOv_9VX6nDsOXKAi0Jvm0GrpDbWYLtJ9Xmbg,7309
+stix_shifter_modules_sysdig-8.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+stix_shifter_modules_sysdig-8.0.2.dist-info/top_level.txt,sha256=NX-VgUOr8fI-lMXHt3gtjfsEn1UPaGAVszt6Z_CTp2s,21
+stix_shifter_modules_sysdig-8.0.2.dist-info/RECORD,,

stix_shifter_modules_sysdig-8.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

stix_shifter_modules_sysdig-8.0.2.dist-info/licenses/AUTHORS.md

@@ -0,0 +1,23 @@
+# Project Authors
+
+This project is a collaborative effort between the [OCA Project](https://opencybersecurityalliance.org/) and IBM Canada. It has been developed and maintained by a dedicated team of contributors over time.
+
+## Current Maintainer & Developer
+
+**Morgan Nolan** – Developer (Maintainer) – Cork, Ireland (IBM)
+
+## Former Maintainers & Developers – IBM, Fredericton (NB), Canada
+
+We gratefully acknowledge the contributions of former IBM Connector Factory maintainers and developers who helped shape this project:
+
+- **Derek Rushton** – Developer (Former Maintainer)  
+- **Eric Larsen** – Developer (Former Maintainer)  
+- **Ben Craig** – Developer (Former Maintainer)  
+- **Danny Elliot** – Developer (Maintainer)  
+- **Md Azam** – Developer (Maintainer)  
+- **Arthur Muradyan** – Developer  
+- **Omkar Gurav** – Developer  
+
+## Special Thanks
+
+We also extend our appreciation to everyone who contributed ideas, bug reports, and feedback to improve the project.