misp-modules 2.4.196__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. misp_modules/__init__.py +379 -0
  2. misp_modules/helpers/__init__.py +1 -0
  3. misp_modules/helpers/cache.py +84 -0
  4. misp_modules/lib/__init__.py +4 -0
  5. misp_modules/lib/_vmray/__init__.py +0 -0
  6. misp_modules/lib/_vmray/parser.py +1417 -0
  7. misp_modules/lib/_vmray/rest_api.py +148 -0
  8. misp_modules/lib/cof2misp/LICENSE-2.0.txt +202 -0
  9. misp_modules/lib/cof2misp/__init__.py +0 -0
  10. misp_modules/lib/cof2misp/cof.py +165 -0
  11. misp_modules/lib/joe_mapping.py +114 -0
  12. misp_modules/lib/joe_parser.py +573 -0
  13. misp_modules/lib/lastline_api.py +841 -0
  14. misp_modules/lib/qintel_helper.py +263 -0
  15. misp_modules/lib/stix2misp.py +2080 -0
  16. misp_modules/lib/stix2misp_mapping.py +460 -0
  17. misp_modules/lib/synonymsToTagNames.json +1 -0
  18. misp_modules/lib/vt_graph_parser/__init__.py +8 -0
  19. misp_modules/lib/vt_graph_parser/errors.py +20 -0
  20. misp_modules/lib/vt_graph_parser/helpers/__init__.py +7 -0
  21. misp_modules/lib/vt_graph_parser/helpers/parsers.py +88 -0
  22. misp_modules/lib/vt_graph_parser/helpers/rules.py +304 -0
  23. misp_modules/lib/vt_graph_parser/helpers/wrappers.py +58 -0
  24. misp_modules/lib/vt_graph_parser/importers/__init__.py +7 -0
  25. misp_modules/lib/vt_graph_parser/importers/base.py +98 -0
  26. misp_modules/lib/vt_graph_parser/importers/pymisp_response.py +73 -0
  27. misp_modules/modules/__init__.py +4 -0
  28. misp_modules/modules/action_mod/__init__.py +1 -0
  29. misp_modules/modules/action_mod/_utils/__init__.py +1 -0
  30. misp_modules/modules/action_mod/_utils/utils.py +70 -0
  31. misp_modules/modules/action_mod/mattermost.py +113 -0
  32. misp_modules/modules/action_mod/slack.py +96 -0
  33. misp_modules/modules/action_mod/testaction.py +68 -0
  34. misp_modules/modules/expansion/__init__.py +36 -0
  35. misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT +27 -0
  36. misp_modules/modules/expansion/_dnsdb_query/LICENSE +202 -0
  37. misp_modules/modules/expansion/_dnsdb_query/README.md +162 -0
  38. misp_modules/modules/expansion/_dnsdb_query/__init__.py +0 -0
  39. misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py +327 -0
  40. misp_modules/modules/expansion/_ransomcoindb/__init__.py +0 -0
  41. misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py +96 -0
  42. misp_modules/modules/expansion/_vulnerability_parser/__init__.py +0 -0
  43. misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py +118 -0
  44. misp_modules/modules/expansion/abuseipdb.py +146 -0
  45. misp_modules/modules/expansion/apiosintds.py +365 -0
  46. misp_modules/modules/expansion/apivoid.py +133 -0
  47. misp_modules/modules/expansion/assemblyline_query.py +179 -0
  48. misp_modules/modules/expansion/assemblyline_submit.py +100 -0
  49. misp_modules/modules/expansion/backscatter_io.py +84 -0
  50. misp_modules/modules/expansion/btc_scam_check.py +54 -0
  51. misp_modules/modules/expansion/btc_steroids.py +238 -0
  52. misp_modules/modules/expansion/censys_enrich.py +287 -0
  53. misp_modules/modules/expansion/circl_passivedns.py +85 -0
  54. misp_modules/modules/expansion/circl_passivessl.py +113 -0
  55. misp_modules/modules/expansion/clamav.py +129 -0
  56. misp_modules/modules/expansion/cluster25_expand.py +239 -0
  57. misp_modules/modules/expansion/countrycode.py +69 -0
  58. misp_modules/modules/expansion/cpe.py +140 -0
  59. misp_modules/modules/expansion/crowdsec.py +235 -0
  60. misp_modules/modules/expansion/crowdstrike_falcon.py +148 -0
  61. misp_modules/modules/expansion/cuckoo_submit.py +161 -0
  62. misp_modules/modules/expansion/cve.py +57 -0
  63. misp_modules/modules/expansion/cve_advanced.py +185 -0
  64. misp_modules/modules/expansion/cytomic_orion.py +196 -0
  65. misp_modules/modules/expansion/dbl_spamhaus.py +78 -0
  66. misp_modules/modules/expansion/dns.py +71 -0
  67. misp_modules/modules/expansion/docx_enrich.py +71 -0
  68. misp_modules/modules/expansion/domaintools.py +290 -0
  69. misp_modules/modules/expansion/eql.py +91 -0
  70. misp_modules/modules/expansion/eupi.py +86 -0
  71. misp_modules/modules/expansion/extract_url_components.py +83 -0
  72. misp_modules/modules/expansion/farsight_passivedns.py +243 -0
  73. misp_modules/modules/expansion/geoip_asn.py +75 -0
  74. misp_modules/modules/expansion/geoip_city.py +75 -0
  75. misp_modules/modules/expansion/geoip_country.py +73 -0
  76. misp_modules/modules/expansion/google_safe_browsing.py +86 -0
  77. misp_modules/modules/expansion/google_search.py +57 -0
  78. misp_modules/modules/expansion/google_threat_intelligence.py +453 -0
  79. misp_modules/modules/expansion/greynoise.py +346 -0
  80. misp_modules/modules/expansion/hashdd.py +53 -0
  81. misp_modules/modules/expansion/hashlookup.py +118 -0
  82. misp_modules/modules/expansion/hibp.py +62 -0
  83. misp_modules/modules/expansion/html_to_markdown.py +63 -0
  84. misp_modules/modules/expansion/hyasinsight.py +881 -0
  85. misp_modules/modules/expansion/intel471.py +73 -0
  86. misp_modules/modules/expansion/intelmq_eventdb.py.experimental +67 -0
  87. misp_modules/modules/expansion/ip2locationio.py +90 -0
  88. misp_modules/modules/expansion/ipasn.py +70 -0
  89. misp_modules/modules/expansion/ipinfo.py +112 -0
  90. misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py +633 -0
  91. misp_modules/modules/expansion/iprep.py +98 -0
  92. misp_modules/modules/expansion/jinja_template_rendering.py +54 -0
  93. misp_modules/modules/expansion/joesandbox_query.py +91 -0
  94. misp_modules/modules/expansion/joesandbox_submit.py +147 -0
  95. misp_modules/modules/expansion/lastline_query.py +148 -0
  96. misp_modules/modules/expansion/lastline_submit.py +180 -0
  97. misp_modules/modules/expansion/macaddress_io.py +130 -0
  98. misp_modules/modules/expansion/macvendors.py +55 -0
  99. misp_modules/modules/expansion/malshare_upload.py +107 -0
  100. misp_modules/modules/expansion/malwarebazaar.py +67 -0
  101. misp_modules/modules/expansion/mcafee_insights_enrich.py +249 -0
  102. misp_modules/modules/expansion/mmdb_lookup.py +138 -0
  103. misp_modules/modules/expansion/module.py.skeleton +44 -0
  104. misp_modules/modules/expansion/mwdb.py +152 -0
  105. misp_modules/modules/expansion/ocr_enrich.py +74 -0
  106. misp_modules/modules/expansion/ods_enrich.py +71 -0
  107. misp_modules/modules/expansion/odt_enrich.py +61 -0
  108. misp_modules/modules/expansion/onyphe.py +243 -0
  109. misp_modules/modules/expansion/onyphe_full.py +387 -0
  110. misp_modules/modules/expansion/otx.py +167 -0
  111. misp_modules/modules/expansion/passive_ssh.py +150 -0
  112. misp_modules/modules/expansion/passivetotal.py +358 -0
  113. misp_modules/modules/expansion/pdf_enrich.py +58 -0
  114. misp_modules/modules/expansion/pptx_enrich.py +65 -0
  115. misp_modules/modules/expansion/qintel_qsentry.py +229 -0
  116. misp_modules/modules/expansion/qrcode.py +99 -0
  117. misp_modules/modules/expansion/ransomcoindb.py +82 -0
  118. misp_modules/modules/expansion/rbl.py +126 -0
  119. misp_modules/modules/expansion/recordedfuture.py +544 -0
  120. misp_modules/modules/expansion/reversedns.py +77 -0
  121. misp_modules/modules/expansion/securitytrails.py +571 -0
  122. misp_modules/modules/expansion/shodan.py +244 -0
  123. misp_modules/modules/expansion/sigma_queries.py +61 -0
  124. misp_modules/modules/expansion/sigma_syntax_validator.py +49 -0
  125. misp_modules/modules/expansion/sigmf_expand.py +303 -0
  126. misp_modules/modules/expansion/socialscan.py +108 -0
  127. misp_modules/modules/expansion/sophoslabs_intelix.py +146 -0
  128. misp_modules/modules/expansion/sourcecache.py +57 -0
  129. misp_modules/modules/expansion/stairwell.py +156 -0
  130. misp_modules/modules/expansion/stix2_pattern_syntax_validator.py +56 -0
  131. misp_modules/modules/expansion/threatcrowd.py +170 -0
  132. misp_modules/modules/expansion/threatfox.py +75 -0
  133. misp_modules/modules/expansion/threatminer.py +152 -0
  134. misp_modules/modules/expansion/triage_submit.py +119 -0
  135. misp_modules/modules/expansion/trustar_enrich.py +235 -0
  136. misp_modules/modules/expansion/urlhaus.py +164 -0
  137. misp_modules/modules/expansion/urlscan.py +269 -0
  138. misp_modules/modules/expansion/variotdbs.py +133 -0
  139. misp_modules/modules/expansion/virustotal.py +311 -0
  140. misp_modules/modules/expansion/virustotal_public.py +272 -0
  141. misp_modules/modules/expansion/virustotal_upload.py +91 -0
  142. misp_modules/modules/expansion/vmray_submit.py +161 -0
  143. misp_modules/modules/expansion/vmware_nsx.py +628 -0
  144. misp_modules/modules/expansion/vulndb.py +292 -0
  145. misp_modules/modules/expansion/vulnerability_lookup.py +324 -0
  146. misp_modules/modules/expansion/vulners.py +81 -0
  147. misp_modules/modules/expansion/vysion.py +221 -0
  148. misp_modules/modules/expansion/whois.py +72 -0
  149. misp_modules/modules/expansion/whoisfreaks.py +232 -0
  150. misp_modules/modules/expansion/wiki.py +58 -0
  151. misp_modules/modules/expansion/xforceexchange.py +189 -0
  152. misp_modules/modules/expansion/xlsx_enrich.py +63 -0
  153. misp_modules/modules/expansion/yara_query.py +70 -0
  154. misp_modules/modules/expansion/yara_syntax_validator.py +49 -0
  155. misp_modules/modules/expansion/yeti.py +196 -0
  156. misp_modules/modules/export_mod/__init__.py +3 -0
  157. misp_modules/modules/export_mod/cef_export.py +93 -0
  158. misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py +150 -0
  159. misp_modules/modules/export_mod/defender_endpoint_export.py +141 -0
  160. misp_modules/modules/export_mod/goamlexport.py +255 -0
  161. misp_modules/modules/export_mod/liteexport.py +97 -0
  162. misp_modules/modules/export_mod/mass_eql_export.py +106 -0
  163. misp_modules/modules/export_mod/nexthinkexport.py +131 -0
  164. misp_modules/modules/export_mod/osqueryexport.py +125 -0
  165. misp_modules/modules/export_mod/pdfexport.py +107 -0
  166. misp_modules/modules/export_mod/testexport.py +73 -0
  167. misp_modules/modules/export_mod/threatStream_misp_export.py +114 -0
  168. misp_modules/modules/export_mod/threat_connect_export.py +125 -0
  169. misp_modules/modules/export_mod/virustotal_collections.py +141 -0
  170. misp_modules/modules/export_mod/vt_graph.py +120 -0
  171. misp_modules/modules/export_mod/yara_export.py +290 -0
  172. misp_modules/modules/import_mod/__init__.py +22 -0
  173. misp_modules/modules/import_mod/cof2misp.py +264 -0
  174. misp_modules/modules/import_mod/csvimport.py +319 -0
  175. misp_modules/modules/import_mod/cuckooimport.py +749 -0
  176. misp_modules/modules/import_mod/email_import.py +294 -0
  177. misp_modules/modules/import_mod/goamlimport.py +188 -0
  178. misp_modules/modules/import_mod/import_blueprint.py +93 -0
  179. misp_modules/modules/import_mod/joe_import.py +74 -0
  180. misp_modules/modules/import_mod/lastline_import.py +160 -0
  181. misp_modules/modules/import_mod/mispjson.py +73 -0
  182. misp_modules/modules/import_mod/ocr.py +122 -0
  183. misp_modules/modules/import_mod/openiocimport.py +99 -0
  184. misp_modules/modules/import_mod/taxii21.py +383 -0
  185. misp_modules/modules/import_mod/testimport.py +73 -0
  186. misp_modules/modules/import_mod/threatanalyzer_import.py +552 -0
  187. misp_modules/modules/import_mod/url_import.py +97 -0
  188. misp_modules/modules/import_mod/vmray_import.py +96 -0
  189. misp_modules/modules/import_mod/vmray_summary_json_import.py +87 -0
  190. misp_modules-2.4.196.dist-info/LICENSE +661 -0
  191. misp_modules-2.4.196.dist-info/METADATA +281 -0
  192. misp_modules-2.4.196.dist-info/RECORD +194 -0
  193. misp_modules-2.4.196.dist-info/WHEEL +4 -0
  194. misp_modules-2.4.196.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,118 @@
1
+ import json
2
+ from pymisp import MISPAttribute, MISPEvent, MISPObject
3
+
4
+
5
+ class VulnerabilityMapping:
6
+ __variot_data_mapping = {
7
+ 'credits': 'credit',
8
+ 'description': 'description',
9
+ 'title': 'summary'
10
+ }
11
+ __variot_flat_mapping = {
12
+ 'cve': 'id', 'id': 'id'
13
+ }
14
+
15
+ @classmethod
16
+ def exploit_mapping(cls) -> dict:
17
+ return cls.__exploit_mapping
18
+
19
+ @classmethod
20
+ def exploit_multiple_mapping(cls) -> dict:
21
+ return cls.__exploit_multiple_mapping
22
+
23
+ @classmethod
24
+ def variot_data_mapping(cls) -> dict:
25
+ return cls.__variot_data_mapping
26
+
27
+ @classmethod
28
+ def variot_flat_mapping(cls) -> dict:
29
+ return cls.__variot_flat_mapping
30
+
31
+
32
+ class VulnerabilityParser:
33
+ def __init__(self, attribute: dict):
34
+ misp_attribute = MISPAttribute()
35
+ misp_attribute.from_dict(**attribute)
36
+ misp_event = MISPEvent()
37
+ misp_event.add_attribute(**misp_attribute)
38
+ self.__misp_attribute = misp_attribute
39
+ self.__misp_event = misp_event
40
+
41
+ @property
42
+ def misp_attribute(self):
43
+ return self.__misp_attribute
44
+
45
+ @property
46
+ def misp_event(self):
47
+ return self.__misp_event
48
+
49
+ def get_results(self) -> dict:
50
+ event = json.loads(self.misp_event.to_json())
51
+ return {
52
+ 'results': {
53
+ key: value for key, value in event.items()
54
+ if key in ('Attribute', 'Object')
55
+ }
56
+ }
57
+
58
+ def _parse_variot_description(self, query_results):
59
+ vulnerability_object = MISPObject('vulnerability')
60
+ for field, relation in self.mapping.variot_flat_mapping().items():
61
+ if query_results.get(field):
62
+ vulnerability_object.add_attribute(
63
+ relation, query_results[field]
64
+ )
65
+ for field, relation in self.mapping.variot_data_mapping().items():
66
+ if query_results.get(field, {}).get('data'):
67
+ vulnerability_object.add_attribute(
68
+ relation, query_results[field]['data']
69
+ )
70
+ if query_results.get('configurations', {}).get('data'):
71
+ for configuration in query_results['configurations']['data']:
72
+ for node in configuration['nodes']:
73
+ for cpe_match in node['cpe_match']:
74
+ if cpe_match['vulnerable']:
75
+ vulnerability_object.add_attribute(
76
+ 'vulnerable-configuration',
77
+ cpe_match['cpe23Uri']
78
+ )
79
+ if query_results.get('cvss', {}).get('data'):
80
+ cvss = {}
81
+ for cvss_data in query_results['cvss']['data']:
82
+ for cvss_v3 in cvss_data['cvssV3']:
83
+ cvss[float(cvss_v3['trust'])] = cvss_v3
84
+ if cvss:
85
+ cvss = cvss[max(cvss)]
86
+ vulnerability_object.add_attribute(
87
+ 'cvss-score', cvss['baseScore']
88
+ )
89
+ vulnerability_object.add_attribute(
90
+ 'cvss-string', cvss['vectorString']
91
+ )
92
+ if query_results.get('references', {}).get('data'):
93
+ for reference in query_results['references']['data']:
94
+ vulnerability_object.add_attribute(
95
+ 'references', reference['url']
96
+ )
97
+ if query_results.get('sources_release_date', {}).get('data'):
98
+ for release_date in query_results['sources_release_date']['data']:
99
+ if release_date['db'] != 'NVD':
100
+ continue
101
+ if release_date['id'] == self.misp_attribute.value:
102
+ vulnerability_object.add_attribute(
103
+ 'published', release_date['date']
104
+ )
105
+ break
106
+ if query_results.get('sources_update_date', {}).get('data'):
107
+ for update_date in query_results['sources_update_date']['data']:
108
+ if update_date['db'] != 'NVD':
109
+ continue
110
+ if update_date['id'] == self.misp_attribute.value:
111
+ vulnerability_object.add_attribute(
112
+ 'modified', update_date['date']
113
+ )
114
+ break
115
+ vulnerability_object.add_reference(
116
+ self.misp_attribute.uuid, 'related-to'
117
+ )
118
+ self.misp_event.add_object(vulnerability_object)
@@ -0,0 +1,146 @@
1
+ import requests
2
+ import json
3
+ from pymisp import MISPObject, MISPAttribute, MISPEvent
4
+ from . import check_input_attribute, checking_error, standard_error_message
5
+ import dns.resolver
6
+
7
+ misperrors = {'error': 'Error'}
8
+ mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain', 'domain|ip'], 'format': 'misp_standard'}
9
+ moduleinfo = {
10
+ 'version': '0.1',
11
+ 'author': 'Stephanie S',
12
+ 'description': 'AbuseIPDB MISP expansion module',
13
+ 'module-type': ['expansion', 'hover'],
14
+ 'name': 'Abuse IPDB',
15
+ 'logo': '',
16
+ 'requirements': [],
17
+ 'features': '',
18
+ 'references': [],
19
+ 'input': '',
20
+ 'output': '',
21
+ }
22
+
23
+ moduleconfig = ['api_key', 'max_age_in_days', 'abuse_threshold']
24
+
25
+ def get_ip(request):
26
+ # Need to get the ip from the domain
27
+ resolver = dns.resolver.Resolver()
28
+ resolver.timeout = 2
29
+ resolver.lifetime = 2
30
+
31
+ try:
32
+ ip = resolver.query(request["attribute"]["value"], 'A')
33
+ return ip
34
+ except dns.resolver.NXDOMAIN:
35
+ misperrors['error'] = "NXDOMAIN"
36
+ return misperrors
37
+ except dns.exception.Timeout:
38
+ misperrors['error'] = "Timeout"
39
+ return misperrors
40
+ except Exception:
41
+ misperrors['error'] = "DNS resolving error"
42
+ return misperrors
43
+
44
+ def handler(q=False):
45
+ if q is False:
46
+ return False
47
+ request = json.loads(q)
48
+
49
+ if "config" not in request or "api_key" not in request["config"]:
50
+ return {"error": "AbuseIPDB API key is missing"}
51
+ if "max_age_in_days" not in request["config"]:
52
+ return {"error": "AbuseIPDB max age in days is missing"}
53
+ if "abuse_threshold" not in request["config"]:
54
+ return {"error": "AbuseIPDB abuse threshold is missing"}
55
+ if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')):
56
+ return {'error': f'{standard_error_message}, {checking_error}.'}
57
+ if request['attribute']['type'] not in mispattributes['input']:
58
+ return {'error': 'Unsupported attribute type.'}
59
+
60
+ if (request['attribute']['type'] == 'hostname' or request['attribute']['type'] == 'domain' or request['attribute']['type'] == 'domain|ip'):
61
+ ip = get_ip(request)[0]
62
+
63
+ else:
64
+ ip = request["attribute"]["value"]
65
+
66
+ api_key = request["config"]["api_key"]
67
+ max_age_in_days = request["config"]["max_age_in_days"]
68
+ api_endpoint = 'https://api.abuseipdb.com/api/v2/check'
69
+ querystring = {
70
+ 'ipAddress': ip,
71
+ 'maxAgeInDays': max_age_in_days
72
+ }
73
+ headers = {
74
+ 'Accept': 'application/json',
75
+ 'key': api_key
76
+ }
77
+ r = {"results": []}
78
+
79
+ response = requests.request(method='GET', url=api_endpoint, headers=headers, params=querystring)
80
+
81
+ if (response.status_code == 200):
82
+ response_json = json.loads(response.text)
83
+ is_whitelisted = response_json['data']['isWhitelisted']
84
+ is_tor = response_json['data']['isTor']
85
+ is_public = response_json['data']['isPublic']
86
+ abuse_confidence_score = response_json['data']['abuseConfidenceScore']
87
+
88
+ abuse_threshold = request["config"]["abuse_threshold"]
89
+
90
+ if (request["config"]["abuse_threshold"] is not None):
91
+ abuse_threshold = request["config"]["abuse_threshold"]
92
+ else:
93
+ abuse_threshold = 70
94
+
95
+ if (is_whitelisted == False):
96
+ is_whitelisted = 0
97
+ if (is_tor == False):
98
+ is_tor = 0
99
+ if (is_public == False):
100
+ is_public = 0
101
+ if (abuse_confidence_score == None):
102
+ abuse_confidence_score = 0
103
+
104
+ if (response_json.get("errors")):
105
+ return {'error': 'AbuseIPDB error, check logs'}
106
+ else:
107
+ event = MISPEvent()
108
+ obj = MISPObject('abuseipdb')
109
+ event.add_attribute(**request['attribute'])
110
+
111
+ if int(abuse_confidence_score) >= int(abuse_threshold):
112
+ malicious_attribute = obj.add_attribute('is-malicious', **{'type': 'boolean', 'value': 1})
113
+ malicious_attribute.add_tag(f'ioc:artifact-state="malicious"')
114
+ else:
115
+ malicious_attribute = obj.add_attribute('is-malicious', **{'type': 'boolean', 'value': 0})
116
+ malicious_attribute.add_tag(f'ioc:artifact-state="not-malicious"')
117
+
118
+ if is_whitelisted is not None:
119
+ obj.add_attribute('is-whitelisted', **{'type': 'boolean', 'value': is_whitelisted})
120
+ obj.add_attribute('is-tor', **{'type': 'boolean', 'value': is_tor})
121
+ obj.add_attribute('is-public', **{'type': 'boolean', 'value': is_public})
122
+ obj.add_attribute('abuse-confidence-score', **{'type': 'counter', 'value': abuse_confidence_score})
123
+ obj.add_reference(request['attribute']['uuid'], "describes")
124
+ event.add_object(obj)
125
+
126
+ # Avoid serialization issue
127
+ event = json.loads(event.to_json())
128
+
129
+ r['results'] = {'Object': event['Object'], 'Attribute': event['Attribute']}
130
+ return r
131
+
132
+ else:
133
+ try:
134
+ response_json = json.loads(response.text)
135
+ if (response_json['errors']):
136
+ return {"error": "API not reachable, status code: " + str(response.status_code) + " " + str(response_json['errors'][0]['detail'])}
137
+ except:
138
+ pass
139
+ return {"error": "API not reachable, status code: " + str(response.status_code)}
140
+
141
+ def introspection():
142
+ return mispattributes
143
+
144
+ def version():
145
+ moduleinfo['config'] = moduleconfig
146
+ return moduleinfo
@@ -0,0 +1,365 @@
1
+ import json
2
+ import logging
3
+ import sys
4
+ import os
5
+ from apiosintDS import apiosintDS
6
+
7
+ log = logging.getLogger('apiosintDS')
8
+ log.setLevel(logging.DEBUG)
9
+ apiodbg = logging.StreamHandler(sys.stdout)
10
+ apiodbg.setLevel(logging.DEBUG)
11
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
12
+ apiodbg.setFormatter(formatter)
13
+ log.addHandler(apiodbg)
14
+
15
+ misperrors = {'error': 'Error'}
16
+
17
+ mispattributes = {'input': ["domain", "domain|ip", "hostname", "ip-dst", "ip-src", "ip-dst|port", "ip-src|port", "url",
18
+ "md5", "sha1", "sha256", "filename|md5", "filename|sha1", "filename|sha256"],
19
+ 'output': ["domain", "ip-dst", "url", "comment", "md5", "sha1", "sha256", "link", "text"]
20
+ }
21
+
22
+ moduleinfo = {
23
+ 'version': '0.2',
24
+ 'author': 'Davide Baglieri aka davidonzo',
25
+ 'description': 'On demand query API for OSINT.digitalside.it project.',
26
+ 'module-type': ['expansion', 'hover'],
27
+ 'name': 'OSINT DigitalSide',
28
+ 'logo': '',
29
+ 'requirements': ['The apiosintDS python library to query the OSINT.digitalside.it API.'],
30
+ 'features': 'The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.\n\nThe result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls.\n\nFurthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it',
31
+ 'references': ['https://osint.digitalside.it/#About'],
32
+ 'input': 'A domain, ip, url or hash attribute.',
33
+ 'output': 'Hashes and urls resulting from the query to OSINT.digitalside.it',
34
+ }
35
+
36
+ moduleconfig = ['STIX2_details', 'import_related', 'cache', 'cache_directory', 'cache_timeout_h', 'local_directory']
37
+
38
+
39
+ def handler(q=False):
40
+ if q is False:
41
+ return False
42
+ request = json.loads(q)
43
+ tosubmit = []
44
+ if request.get('domain'):
45
+ tosubmit.append(request['domain'])
46
+ elif request.get('domain|ip'):
47
+ tosubmit.append(request['domain|ip'].split('|')[0])
48
+ tosubmit.append(request['domain|ip'].split('|')[1])
49
+ elif request.get('hostname'):
50
+ tosubmit.append(request['hostname'])
51
+ elif request.get('ip-dst'):
52
+ tosubmit.append(request['ip-dst'])
53
+ elif request.get('ip-src'):
54
+ tosubmit.append(request['ip-src'])
55
+ elif request.get('ip-dst|port'):
56
+ tosubmit.append(request['ip-dst|port'].split('|')[0])
57
+ elif request.get('ip-src|port'):
58
+ tosubmit.append(request['ip-src|port'].split('|')[0])
59
+ elif request.get('url'):
60
+ tosubmit.append(request['url'])
61
+ elif request.get('md5'):
62
+ tosubmit.append(request['md5'])
63
+ elif request.get('sha1'):
64
+ tosubmit.append(request['sha1'])
65
+ elif request.get('sha256'):
66
+ tosubmit.append(request['sha256'])
67
+ elif request.get('filename|md5'):
68
+ tosubmit.append(request['filename|md5'].split('|')[1])
69
+ elif request.get('filename|sha1'):
70
+ tosubmit.append(request['filename|sha1'].split('|')[1])
71
+ elif request.get('filename|sha256'):
72
+ tosubmit.append(request['filename|sha256'].split('|')[1])
73
+ else:
74
+ return False
75
+
76
+ persistent = 0
77
+ if request.get('persistent'):
78
+ persistent = request["persistent"]
79
+
80
+ submitcache = False
81
+ submitcache_directory = False
82
+ submitcache_timeout = False
83
+ submit_stix = False
84
+ import_related = False
85
+ sumbit_localdirectory = False
86
+
87
+ r = {"results": []}
88
+
89
+ if request.get('config'):
90
+
91
+ if request['config'].get('cache') and request['config']['cache'].lower() == "yes":
92
+ submitcache = True
93
+
94
+ if request['config'].get('import_related') and request['config']['import_related'].lower() == "yes":
95
+ import_related = True
96
+
97
+ if request['config'].get('STIX2_details') and request['config']['STIX2_details'].lower() == "yes":
98
+ submit_stix = True
99
+
100
+ if request['config'].get('cache_timeout_h') and len(request['config']['cache_timeout_h']) > 0:
101
+ submitcache_timeout = int(request['config'].get('cache_timeout_h'))
102
+
103
+ localdirectory = request['config'].get('local_directory')
104
+ if localdirectory and len(localdirectory) > 0:
105
+ if os.access(localdirectory, os.R_OK):
106
+ sumbit_localdirectory = localdirectory
107
+ WarningMSG = "Local directory OK! Ignoring cache configuration..."
108
+ log.debug(str(WarningMSG))
109
+ submitcache = False
110
+ sumbitcache_titmeout = False
111
+ submitcache_directory = False
112
+ else:
113
+ ErrorMSG = "Unable to read local 'Threat-Intel' directory ("+localdirectory+"). Please, check your configuration and retry."
114
+ log.debug(str(ErrorMSG))
115
+ misperrors['error'] = ErrorMSG
116
+ return misperrors
117
+
118
+ if submitcache:
119
+ cache_directory = request['config'].get('cache_directory')
120
+ if cache_directory and len(cache_directory) > 0:
121
+ if os.access(cache_directory, os.W_OK):
122
+ submitcache_directory = cache_directory
123
+ else:
124
+ ErrorMSG = "Cache directory is not writable. Please fix it before."
125
+ log.debug(str(ErrorMSG))
126
+ misperrors['error'] = ErrorMSG
127
+ return misperrors
128
+ else:
129
+ ErrorMSG = "Value for Plugin.Enrichment_apiosintds_cache_directory is empty but cache option is enabled as recommended. Please set a writable cache directory in plugin settings."
130
+ log.debug(str(ErrorMSG))
131
+ misperrors['error'] = ErrorMSG
132
+ return misperrors
133
+ else:
134
+ if sumbit_localdirectory == False:
135
+ log.debug("Cache option is set to " + str(submitcache) + ". You are not using the internal cache system and this is NOT recommended!")
136
+ log.debug("Please, consider to turn on the cache setting it to 'Yes' and specifing a writable directory for the cache directory option.")
137
+ try:
138
+ response = apiosintDS.request(entities=tosubmit, stix=submit_stix, cache=submitcache, cachedirectory=submitcache_directory, cachetimeout=submitcache_timeout, verbose=True, localdirectory=sumbit_localdirectory)
139
+ r["results"] += apiosintParserHover(persistent, response, import_related, submit_stix)
140
+ return r
141
+ except Exception as e:
142
+ log.exception("Could not process apiosintDS")
143
+ return {'error': str(e)}
144
+
145
+
146
+ def apiosintParserHover(ispersistent, response, import_related, stix):
147
+ apiosinttype = ['hash', 'ip', 'url', 'domain']
148
+ line = "##############################################"
149
+ linedot = "--------------------------------------------------------------------"
150
+ linedotty = "-------------------"
151
+ ret = []
152
+ retHover = []
153
+ if isinstance(response, dict):
154
+ for key in response:
155
+ if key in apiosinttype:
156
+ for item in response[key]["items"]:
157
+ if item["response"]:
158
+ comment = "IoC '"+item["item"] + "' found in OSINT.DigitaiSide.it repository. List file: "+response[key]["list"]["file"]+". List date: " + response[key]["list"]["date"]
159
+ commentH = "IoC '"+item["item"] + "' found in OSINT.DigitaiSide.it repository."
160
+ CommentHDate = "List file: "+response[key]["list"]["file"]+". Date list: " + response[key]["list"]["date"]
161
+ ret.append({"types": ["text"], "values": [comment]})
162
+
163
+ retHover.append({"types": ["text"], "values": [commentH]})
164
+ retHover.append({"types": ["text"], "values": [CommentHDate]})
165
+ retHover.append({"types": ["text"], "values": [line]})
166
+
167
+ if key in ["url", "hash"]:
168
+ if "hashes" in item:
169
+ headhash = "Hashes set"
170
+ retHover.append({"types": ["text"], "values": [headhash]})
171
+ if "md5" in item["hashes"].keys():
172
+ ret.append({"types": ["md5"], "values": [item["hashes"]["md5"]], "comment": "Related to: " + item["item"]})
173
+
174
+ strmd5 = "MD5: "+item["hashes"]["md5"]
175
+ retHover.append({"types": ["text"], "values": [strmd5]})
176
+
177
+ if "sha1" in item["hashes"].keys():
178
+ ret.append({"types": ["sha1"], "values": [item["hashes"]["sha1"]], "comment": "Related to: " + item["item"]})
179
+
180
+ strsha1 = "SHA1: "+item["hashes"]["sha1"]
181
+ retHover.append({"types": ["text"], "values": [strsha1]})
182
+
183
+ if "sha256" in item["hashes"].keys():
184
+ ret.append({"types": ["sha256"], "values": [item["hashes"]["sha256"]], "comment": "Related to: " + item["item"]})
185
+
186
+ strsha256 = "SHA256: "+item["hashes"]["sha256"]
187
+ retHover.append({"types": ["text"], "values": [strsha256]})
188
+
189
+ if "online_reports" in item:
190
+ headReports = "Online Reports (availability depends on retention)"
191
+ retHover.append({"types": ["text"], "values": [linedot]})
192
+ retHover.append({"types": ["text"], "values": [headReports]})
193
+ onlierepor = item["online_reports"]
194
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_EVENT"]], "comment": "MISP Event related to: " + item["item"]})
195
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_CSV"]], "comment": "MISP CSV related to: " + item["item"]})
196
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["OSINTDS_REPORT"]], "comment": "DigitalSide report related to: " + item["item"]})
197
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["STIX"]], "comment": "STIX2 report related to: " + item["item"]})
198
+
199
+ MISPEVENT = "MISP Event => "+onlierepor["MISP_EVENT"]
200
+ MISPCSV = "MISP CSV => "+onlierepor["MISP_CSV"]
201
+ OSINTDS = "DigitalSide report => "+onlierepor["OSINTDS_REPORT"]
202
+ STIX = "STIX report => "+onlierepor["STIX"]
203
+
204
+ retHover.append({"types": ["text"], "values": [MISPEVENT]})
205
+ retHover.append({"types": ["text"], "values": [MISPCSV]})
206
+ retHover.append({"types": ["text"], "values": [OSINTDS]})
207
+ retHover.append({"types": ["text"], "values": [STIX]})
208
+
209
+ if stix and onlierepor:
210
+ if "STIXDETAILS" in onlierepor:
211
+ retHover.append({"types": ["text"], "values": [linedot]})
212
+ headStix = "STIX2 report details"
213
+ stixobj = onlierepor["STIXDETAILS"]
214
+ stxdet = "TLP:"+stixobj["tlp"]+" | Observation: "+str(stixobj["number_observed"])+" | First seen: "+stixobj["first_observed"]+" | First seen: "+stixobj["last_observed"]
215
+ ret.append({"types": ["comment"], "values": [stxdet], "comment": "STIX2 details for: " + item["item"]})
216
+ retHover.append({"types": ["text"], "values": [headStix]})
217
+ retHover.append({"types": ["text"], "values": [stxdet]})
218
+
219
+
220
+ if stixobj["observed_time_frame"] != False:
221
+ obstf = "Observation time frame: "+str(stixobj["observed_time_frame"])
222
+ ret.append({"types": ["comment"], "values": [obstf], "comment": "STIX2 details for: " + item["item"]})
223
+ retHover.append({"types": ["text"], "values": [obstf]})
224
+
225
+ filename = stixobj["filename"]
226
+ ret.append({"category": "Payload delivery", "types": ["filename"], "values": [filename], "comment": "STIX2 details for: " + item["item"]})
227
+
228
+ Hovefilename = "Filename: "+filename
229
+ retHover.append({"types": ["text"], "values": [Hovefilename]})
230
+
231
+ filesize = stixobj["filesize"]
232
+ ret.append({"types": ["size-in-bytes"], "values": [filesize], "comment": "STIX2 details for: " + item["item"]})
233
+
234
+ Hovefilesize = "Filesize in bytes: "+str(filesize)
235
+ retHover.append({"types": ["text"], "values": [Hovefilesize]})
236
+
237
+ filetype = stixobj["mime_type"]
238
+ ret.append({"category": "Payload delivery", "types": ["mime-type"], "values": [filetype], "comment": "STIX2 details for: " + item["item"]})
239
+
240
+ Hovemime = "Filetype: "+filetype
241
+ retHover.append({"types": ["text"], "values": [Hovemime]})
242
+
243
+ if "virus_total" in stixobj:
244
+ if stixobj["virus_total"] != False:
245
+ VTratio = "VirusTotal Ratio: "+str(stixobj["virus_total"]["vt_detection_ratio"])
246
+ ret.append({"types": ["comment"], "values": [VTratio], "comment": "STIX2 details for: " + item["item"]})
247
+ retHover.append({"types": ["text"], "values": [VTratio]})
248
+
249
+ VTReport = str(stixobj["virus_total"]["vt_report"])
250
+ ret.append({"category": "External analysis", "types": ["link"], "values": [VTReport], "comment": "VirusTotal Report for: " + item["item"]})
251
+ if import_related:
252
+ if len(item["related_urls"]) > 0:
253
+ retHover.append({"types": ["text"], "values": [linedot]})
254
+ countRelated = "Related URLS count: "+str(len(item["related_urls"]))
255
+ retHover.append({"types": ["text"], "values": [countRelated]})
256
+ for urls in item["related_urls"]:
257
+ if isinstance(urls, dict):
258
+ itemToInclude = urls["url"]
259
+ ret.append({"types": ["url"], "values": [itemToInclude], "comment": "Download URL for "+urls["hashes"]["md5"]+". Related to: " + item["item"]})
260
+
261
+ retHover.append({"types": ["text"], "values": [linedot]})
262
+ relatedURL = "Related URL "+itemToInclude
263
+ retHover.append({"types": ["text"], "values": [relatedURL]})
264
+
265
+ if "hashes" in urls.keys():
266
+ if "md5" in urls["hashes"].keys():
267
+ ret.append({"types": ["md5"], "values": [urls["hashes"]["md5"]], "comment": "Related to: " + itemToInclude})
268
+
269
+ strmd5 = "MD5: "+urls["hashes"]["md5"]
270
+ retHover.append({"types": ["text"], "values": [strmd5]})
271
+
272
+ if "sha1" in urls["hashes"].keys():
273
+ ret.append({"types": ["sha1"], "values": [urls["hashes"]["sha1"]], "comment": "Related to: " + itemToInclude})
274
+
275
+ strsha1 = "SHA1: "+urls["hashes"]["sha1"]
276
+ retHover.append({"types": ["text"], "values": [strsha1]})
277
+
278
+ if "sha256" in urls["hashes"].keys():
279
+ ret.append({"types": ["sha256"], "values": [urls["hashes"]["sha256"]], "comment": "Related to: " + itemToInclude})
280
+
281
+ strsha256 = "SHA256: "+urls["hashes"]["sha256"]
282
+ retHover.append({"types": ["text"], "values": [strsha256]})
283
+
284
+
285
+ headReports = "Online Reports (availability depends on retention)"
286
+ retHover.append({"types": ["text"], "values": [linedotty]})
287
+ retHover.append({"types": ["text"], "values": [headReports]})
288
+ onlierepor = urls["online_reports"]
289
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_EVENT"]], "comment": "MISP Event related to: " + item["item"]})
290
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_CSV"]], "comment": "MISP CSV related to: " + item["item"]})
291
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["OSINTDS_REPORT"]], "comment": "DigitalSide report related to: " + item["item"]})
292
+ ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["STIX"]], "comment": "STIX2 report related to: " + item["item"]})
293
+
294
+ MISPEVENT = "MISP Event => "+onlierepor["MISP_EVENT"]
295
+ MISPCSV = "MISP CSV => "+onlierepor["MISP_CSV"]
296
+ OSINTDS = "DigitalSide report => "+onlierepor["OSINTDS_REPORT"]
297
+ STIX = "STIX report => "+onlierepor["STIX"]
298
+
299
+ retHover.append({"types": ["text"], "values": [MISPEVENT]})
300
+ retHover.append({"types": ["text"], "values": [MISPCSV]})
301
+ retHover.append({"types": ["text"], "values": [OSINTDS]})
302
+ retHover.append({"types": ["text"], "values": [STIX]})
303
+
304
+ if stix and onlierepor:
305
+ if "STIXDETAILS" in onlierepor:
306
+ retHover.append({"types": ["text"], "values": [linedotty]})
307
+ headStix = "STIX2 report details"
308
+ stixobj = onlierepor["STIXDETAILS"]
309
+ stxdet = "TLP:"+stixobj["tlp"]+" | Observation: "+str(stixobj["number_observed"])+" | First seen: "+stixobj["first_observed"]+" | First seen: "+stixobj["last_observed"]
310
+ ret.append({"types": ["comment"], "values": [stxdet], "comment": "STIX2 details for: " + item["item"]})
311
+ retHover.append({"types": ["text"], "values": [headStix]})
312
+ retHover.append({"types": ["text"], "values": [stxdet]})
313
+
314
+ if stixobj["observed_time_frame"] != False:
315
+ obstf = "Observation time frame: "+str(stixobj["observed_time_frame"])
316
+ ret.append({"types": ["comment"], "values": [obstf], "comment": "STIX2 details for: " + item["item"]})
317
+ retHover.append({"types": ["text"], "values": [obstf]})
318
+
319
+ filename = stixobj["filename"]
320
+ ret.append({"category": "Payload delivery", "types": ["filename"], "values": [filename], "comment": "STIX2 details for: " + item["item"]})
321
+
322
+ Hovefilename = "Filename: "+filename
323
+ retHover.append({"types": ["text"], "values": [Hovefilename]})
324
+
325
+ filesize = stixobj["filesize"]
326
+ ret.append({"types": ["size-in-bytes"], "values": [filesize], "comment": "STIX2 details for: " + item["item"]})
327
+
328
+ Hovefilesize = "Filesize in bytes: "+str(filesize)
329
+ retHover.append({"types": ["text"], "values": [Hovefilesize]})
330
+
331
+ filetype = stixobj["mime_type"]
332
+ ret.append({"category": "Payload delivery", "types": ["mime-type"], "values": [filetype], "comment": "STIX2 details for: " + item["item"]})
333
+
334
+ Hovemime = "Filetype: "+filetype
335
+ retHover.append({"types": ["text"], "values": [Hovemime]})
336
+
337
+ if "virus_total" in stixobj:
338
+ if stixobj["virus_total"] != False:
339
+ VTratio = "VirusTotal Ratio: "+stixobj["virus_total"]["vt_detection_ratio"]
340
+ ret.append({"types": ["comment"], "values": [VTratio], "comment": "STIX2 details for: " + item["item"]})
341
+ retHover.append({"types": ["text"], "values": [VTratio]})
342
+
343
+ VTReport = stixobj["virus_total"]["vt_report"]
344
+ ret.append({"category": "External analysis", "types": ["link"], "values": [VTReport], "comment": "VirusTotal Report for: " + item["item"]})
345
+ else:
346
+ ret.append({"types": ["url"], "values": [urls], "comment": "Download URL for: " + item["item"]})
347
+ urlHover = "URL => "+urls
348
+ retHover.append({"types": ["text"], "values": [urlHover]})
349
+ else:
350
+ notfound = item["item"] + " IS NOT listed by OSINT.digitalside.it. Date list: " + response[key]["list"]["date"]
351
+ ret.append({"types": ["comment"], "values": [notfound]})
352
+ retHover.append({"types": ["comment"], "values": [notfound]})
353
+
354
+ if ispersistent == 0:
355
+ return ret
356
+ return retHover
357
+
358
+
359
+ def introspection():
360
+ return mispattributes
361
+
362
+
363
+ def version():
364
+ moduleinfo['config'] = moduleconfig
365
+ return moduleinfo