parse_ioc 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: parse_ioc
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Parse a list of indicators into a dictionary or JSON structure for programmatic use.
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -9,9 +9,9 @@ Dynamic: license-file
9
9
 
10
10
  # ParseIOC
11
11
 
12
- Parse a list of indicators into a dictionary or JSON structure for programmatic use.
12
+ Parse a list of indicators into a dictionary, JSON structure, or Sqlite database for programmatic use.
13
13
 
14
- Converts a plain list of indicators into a dictionary of:
14
+ Handles:
15
15
  - emails and their domains
16
16
  - URL domains, and username+password and ports if present
17
17
  - ipv4 and ipv6 addresses and networks
@@ -23,7 +23,7 @@ This enables programatic use of an IOC file to quickly query specific values in
23
23
 
24
24
  This tool will support more formats (imphash, ssdeep, ja3+) "eventually".
25
25
 
26
- **Otherwise-unidentifiable strings are categorized as domains.**
26
+ **Otherwise-unidentifiable strings are categorized as "file".**
27
27
 
28
28
  ## Setup
29
29
 
@@ -34,7 +34,7 @@ pip3 install parse_ioc
34
34
  ```
35
35
 
36
36
  ## Usage
37
- `from parse_ioc import ParseIOC, map_fields, parse_multi`
37
+ `from parse_ioc import ParseIOC, map_fields, parse_multi, to_sqlite`
38
38
 
39
39
 
40
40
  ### Categorize a single indicator
@@ -60,22 +60,40 @@ m = map_fields("ioc_examples.txt", "map_ecs.toml")
60
60
  print(json.dumps(m, indent=4))
61
61
  ```
62
62
 
63
- ## Output from running `parse_ioc.py` directly
63
+ ### Create Sqlite database of parsed indicators
64
+ The schema is simply "ioc" and "type"; the dict/JSON keys, are the type values
65
+ ```
66
+ to_sqlite("ioc_examples.txt", "iocs.db")
67
+ ```
68
+
69
+ ## Gotchas
70
+ - Domain names must have a period (`.`) otherwise the string will be considered a file
71
+ - This regex in the domain checker, determines if the parsed domain name drops further down to the "file" bucket or not:
72
+ ```
73
+ file_extensions = re.compile(r"\.(?:exe|dll|msi|bat|cmd|elf|scr|cpl|ps1|vbs|pdf|docx|xlsx|pptx|doc|xls|ppt|rtf|csv|txt|log|xml|zip|rar|7z|tar|gz|iso|img|dmg|cab|png|jpg|jpeg|gif|ico|bmp|svg|mp3|mp4|wav|avi|mov|js|php|css|html|htm|sql|conf|ini|yaml|yml|json)$", re.IGNORECASE)
74
+ ```
75
+
76
+ ## Known Issues
77
+
78
+ - need to streamline and optimize code
79
+ - remove both regex statements created by AI
80
+
81
+ ## Expected sample output from running `parse_ioc.py` directly
64
82
  ```
65
83
  ======================== categorize a single indicator =========================
66
84
  .to_dict: <class 'dict'> {'ioc': '192.168.20.20', 'ioc_type': 'ipv4'}
67
85
  .to_json: <class 'str'> {"ioc": "192.168.20.20", "ioc_type": "ipv4"}
68
- ============ parse a list or file of IOCs into a combined structure ============
86
+ ================ parse a list of IOCs into a combined structure ================
69
87
  {
70
88
  "email": [
71
89
  "bob@email.local"
72
90
  ],
73
91
  "domain": [
74
- "securewebsite.local",
75
- "email.local",
76
92
  "anotherwebsite.local",
93
+ "securewebsite.local",
77
94
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
78
- "website.local"
95
+ "website.local",
96
+ "email.local"
79
97
  ],
80
98
  "port": [
81
99
  9443,
@@ -85,36 +103,33 @@ print(json.dumps(m, indent=4))
85
103
  "username:password"
86
104
  ],
87
105
  "ipv4": [
88
- "192.168.1.1",
89
- "192.168.20.20"
106
+ "192.168.20.20",
107
+ "192.168.1.1"
90
108
  ],
91
109
  "ipv4_network": [
92
110
  "192.168.1.0/24"
93
111
  ]
94
112
  }
113
+ ================ parse a file of IOCs into a combined structure ================
95
114
  {
96
115
  "email": [
97
116
  "bad.username@subdomain.bad.local",
98
117
  "bob@email.local"
99
118
  ],
100
119
  "domain": [
120
+ "anotherwebsite.local",
101
121
  "securewebsite.local",
102
- "email.local",
103
- "bob documents.pdf",
104
- "file.txt",
105
122
  "bad.local",
106
- "not-an-ioc",
107
- "anotherwebsite.local",
108
123
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
109
- "subdomain.bad.local",
110
124
  "website.local",
111
- "aaaaaaaaaaaaaaaa"
125
+ "subdomain.bad.local",
126
+ "email.local"
112
127
  ],
113
128
  "ipv4": [
114
- "8.8.8.8",
115
- "192.168.1.1",
116
129
  "192.168.20.20",
117
- "1.2.3.4"
130
+ "1.2.3.4",
131
+ "8.8.8.8",
132
+ "192.168.1.1"
118
133
  ],
119
134
  "port": [
120
135
  9443,
@@ -124,8 +139,8 @@ print(json.dumps(m, indent=4))
124
139
  "username:password"
125
140
  ],
126
141
  "ipv4_network": [
127
- "8.8.8.0/24",
128
- "192.168.1.0/24"
142
+ "192.168.1.0/24",
143
+ "8.8.8.0/24"
129
144
  ],
130
145
  "ipv6": [
131
146
  "fe80::"
@@ -133,16 +148,20 @@ print(json.dumps(m, indent=4))
133
148
  "file_path_linux": [
134
149
  "/home/bob/file.txt"
135
150
  ],
136
- "unknown": [
137
- "rc:\\users\\bob smith\\desktop\\file.txt:alt.exe"
151
+ "file": [
152
+ "aaaaaaaaaaaaaaaa",
153
+ "file.txt",
154
+ "bob documents.pdf",
155
+ "rc:\\users\\bob smith\\desktop\\file.txt:alt.exe",
156
+ "not-an-ioc"
138
157
  ],
139
158
  "file_path_windows": [
140
- "c:\\users\\bob\\desktop\\test.txt",
141
159
  "c:/users/bob smith/desktop/file.txt:alt.exe",
142
- "c:\\users\\bob smith\\desktop\\file.txt:alt.exe",
160
+ "c:\\users\\bob\\desktop\\test.txt",
143
161
  "file.txt:alt.exe",
144
- "c:\\users\\bob smith\\desktop\\file.txt",
145
- "c:\\users\\bob smith\\desktop\\file2.txt:alt.exe"
162
+ "c:\\users\\bob smith\\desktop\\file.txt:alt.exe",
163
+ "c:\\users\\bob smith\\desktop\\file2.txt:alt.exe",
164
+ "c:\\users\\bob smith\\desktop\\file.txt"
146
165
  ],
147
166
  "md5": [
148
167
  "6cd3556deb0da54bca060b4c39479839"
@@ -170,10 +189,10 @@ print(json.dumps(m, indent=4))
170
189
  {'ioc': 'n--nhk-u63b1cko2lyc6jrwxgom6k.com', 'ioc_type': 'domain'}
171
190
  {'ioc': 'bad.local', 'ioc_type': 'domain'}
172
191
  {'ioc': '/home/bob/file.txt', 'ioc_type': 'file_path_linux'}
173
- {'ioc': 'rc:\\users\\bob smith\\desktop\\file.txt:alt.exe', 'ioc_type': 'unknown'}
174
- {'ioc': 'file.txt', 'ioc_type': 'domain'}
192
+ {'ioc': 'rc:\\users\\bob smith\\desktop\\file.txt:alt.exe', 'ioc_type': 'file'}
193
+ {'ioc': 'file.txt', 'ioc_type': 'file'}
175
194
  {'ioc': 'file.txt:alt.exe', 'ioc_type': 'file_path_windows'}
176
- {'ioc': 'bob documents.pdf', 'ioc_type': 'domain'}
195
+ {'ioc': 'bob documents.pdf', 'ioc_type': 'file'}
177
196
  {'ioc': '/home/bob/file.txt', 'ioc_type': 'file_path_linux'}
178
197
  {'ioc': 'c:\\users\\bob\\desktop\\test.txt', 'ioc_type': 'file_path_windows'}
179
198
  {'ioc': 'c:\\users\\bob smith\\desktop\\file.txt', 'ioc_type': 'file_path_windows'}
@@ -183,8 +202,8 @@ print(json.dumps(m, indent=4))
183
202
  {'ioc': 'c:\\users\\bob smith\\desktop\\file.txt:alt.exe', 'ioc_type': 'file_path_windows'}
184
203
  {'ioc': 'c:\\users\\bob smith\\desktop\\file2.txt:alt.exe', 'ioc_type': 'file_path_windows'}
185
204
  {'ioc': 'c:/users/bob smith/desktop/file.txt:alt.exe', 'ioc_type': 'file_path_windows'}
186
- {'ioc': 'not-an-ioc', 'ioc_type': 'domain'}
187
- {'ioc': 'aaaaaaaaaaaaaaaa', 'ioc_type': 'domain'}
205
+ {'ioc': 'not-an-ioc', 'ioc_type': 'file'}
206
+ {'ioc': 'aaaaaaaaaaaaaaaa', 'ioc_type': 'file'}
188
207
  {'ioc': '6cd3556deb0da54bca060b4c39479839', 'ioc_type': 'md5'}
189
208
  {'ioc': '315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3', 'ioc_type': 'sha256'}
190
209
  {'ioc': 'c1527cd893c124773d811911970c8fe6e857d6df5dc9226bd8a160614c0cd963a4ddea2b94bb7d36021ef9d865d5cea294a82dd49a0bb269f51f6e7a57f79421', 'ioc_type': 'sha512'}
@@ -215,190 +234,166 @@ print(json.dumps(m, indent=4))
215
234
  "bob@email.local"
216
235
  ],
217
236
  "destination.domain": [
237
+ "anotherwebsite.local",
218
238
  "securewebsite.local",
219
- "email.local",
220
- "bob documents.pdf",
221
- "file.txt",
222
239
  "bad.local",
223
- "not-an-ioc",
224
- "anotherwebsite.local",
225
240
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
226
- "subdomain.bad.local",
227
241
  "website.local",
228
- "aaaaaaaaaaaaaaaa"
242
+ "subdomain.bad.local",
243
+ "email.local"
229
244
  ],
230
245
  "url.domain": [
246
+ "anotherwebsite.local",
231
247
  "securewebsite.local",
232
- "email.local",
233
- "bob documents.pdf",
234
- "file.txt",
235
248
  "bad.local",
236
- "not-an-ioc",
237
- "anotherwebsite.local",
238
249
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
239
- "subdomain.bad.local",
240
250
  "website.local",
241
- "aaaaaaaaaaaaaaaa"
251
+ "subdomain.bad.local",
252
+ "email.local"
242
253
  ],
243
254
  "tls.client.server_name": [
255
+ "anotherwebsite.local",
244
256
  "securewebsite.local",
245
- "email.local",
246
- "bob documents.pdf",
247
- "file.txt",
248
257
  "bad.local",
249
- "not-an-ioc",
250
- "anotherwebsite.local",
251
258
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
252
- "subdomain.bad.local",
253
259
  "website.local",
254
- "aaaaaaaaaaaaaaaa"
260
+ "subdomain.bad.local",
261
+ "email.local"
255
262
  ],
256
263
  "dns.question.registered_domain": [
264
+ "anotherwebsite.local",
257
265
  "securewebsite.local",
258
- "email.local",
259
- "bob documents.pdf",
260
- "file.txt",
261
266
  "bad.local",
262
- "not-an-ioc",
263
- "anotherwebsite.local",
264
267
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
265
- "subdomain.bad.local",
266
268
  "website.local",
267
- "aaaaaaaaaaaaaaaa"
269
+ "subdomain.bad.local",
270
+ "email.local"
268
271
  ],
269
272
  "file.origin_referrer_url": [
273
+ "anotherwebsite.local",
270
274
  "securewebsite.local",
271
- "email.local",
272
- "bob documents.pdf",
273
- "file.txt",
274
275
  "bad.local",
275
- "not-an-ioc",
276
- "anotherwebsite.local",
277
276
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
278
- "subdomain.bad.local",
279
277
  "website.local",
280
- "aaaaaaaaaaaaaaaa"
278
+ "subdomain.bad.local",
279
+ "email.local"
281
280
  ],
282
281
  "file.origin_url": [
282
+ "anotherwebsite.local",
283
283
  "securewebsite.local",
284
- "email.local",
285
- "bob documents.pdf",
286
- "file.txt",
287
284
  "bad.local",
288
- "not-an-ioc",
289
- "anotherwebsite.local",
290
285
  "n--nhk-u63b1cko2lyc6jrwxgom6k.com",
291
- "subdomain.bad.local",
292
286
  "website.local",
293
- "aaaaaaaaaaaaaaaa"
287
+ "subdomain.bad.local",
288
+ "email.local"
294
289
  ],
295
290
  "source.ip": [
296
- "8.8.8.8",
297
- "192.168.1.1",
298
291
  "192.168.20.20",
299
292
  "1.2.3.4",
300
- "8.8.8.0/24",
293
+ "8.8.8.8",
294
+ "192.168.1.1",
301
295
  "192.168.1.0/24",
296
+ "8.8.8.0/24",
302
297
  "fe80::"
303
298
  ],
304
299
  "destination.ip": [
305
- "8.8.8.8",
306
- "192.168.1.1",
307
300
  "192.168.20.20",
308
301
  "1.2.3.4",
309
- "8.8.8.0/24",
302
+ "8.8.8.8",
303
+ "192.168.1.1",
310
304
  "192.168.1.0/24",
305
+ "8.8.8.0/24",
311
306
  "fe80::"
312
307
  ],
313
308
  "dns.resolved_ip": [
314
- "8.8.8.8",
315
- "192.168.1.1",
316
309
  "192.168.20.20",
317
310
  "1.2.3.4",
318
- "8.8.8.0/24",
311
+ "8.8.8.8",
312
+ "192.168.1.1",
319
313
  "192.168.1.0/24",
314
+ "8.8.8.0/24",
320
315
  "fe80::"
321
316
  ],
322
317
  "host.ip": [
323
- "8.8.8.8",
324
- "192.168.1.1",
325
318
  "192.168.20.20",
326
319
  "1.2.3.4",
327
- "8.8.8.0/24",
320
+ "8.8.8.8",
321
+ "192.168.1.1",
328
322
  "192.168.1.0/24",
323
+ "8.8.8.0/24",
329
324
  "fe80::"
330
325
  ],
331
326
  "network.forwarded_ip": [
332
- "8.8.8.8",
333
- "192.168.1.1",
334
327
  "192.168.20.20",
335
328
  "1.2.3.4",
336
- "8.8.8.0/24",
329
+ "8.8.8.8",
330
+ "192.168.1.1",
337
331
  "192.168.1.0/24",
332
+ "8.8.8.0/24",
338
333
  "fe80::"
339
334
  ],
340
335
  "related.ip": [
341
- "8.8.8.8",
342
- "192.168.1.1",
343
336
  "192.168.20.20",
344
337
  "1.2.3.4",
345
- "8.8.8.0/24",
338
+ "8.8.8.8",
339
+ "192.168.1.1",
346
340
  "192.168.1.0/24",
341
+ "8.8.8.0/24",
347
342
  "fe80::"
348
343
  ],
349
344
  "client.ip": [
350
- "8.8.8.8",
351
- "192.168.1.1",
352
345
  "192.168.20.20",
353
346
  "1.2.3.4",
354
- "8.8.8.0/24",
347
+ "8.8.8.8",
348
+ "192.168.1.1",
355
349
  "192.168.1.0/24",
350
+ "8.8.8.0/24",
356
351
  "fe80::"
357
352
  ],
358
353
  "server.ip": [
359
- "8.8.8.8",
360
- "192.168.1.1",
361
354
  "192.168.20.20",
362
355
  "1.2.3.4",
363
- "8.8.8.0/24",
356
+ "8.8.8.8",
357
+ "192.168.1.1",
364
358
  "192.168.1.0/24",
359
+ "8.8.8.0/24",
365
360
  "fe80::"
366
361
  ],
367
362
  "server.nat.ip": [
368
- "8.8.8.8",
369
- "192.168.1.1",
370
363
  "192.168.20.20",
371
364
  "1.2.3.4",
372
- "8.8.8.0/24",
365
+ "8.8.8.8",
366
+ "192.168.1.1",
373
367
  "192.168.1.0/24",
368
+ "8.8.8.0/24",
374
369
  "fe80::"
375
370
  ],
376
371
  "threat.enrichments.indicator.ip": [
377
- "8.8.8.8",
378
- "192.168.1.1",
379
372
  "192.168.20.20",
380
373
  "1.2.3.4",
381
- "8.8.8.0/24",
374
+ "8.8.8.8",
375
+ "192.168.1.1",
382
376
  "192.168.1.0/24",
377
+ "8.8.8.0/24",
383
378
  "fe80::"
384
379
  ],
385
380
  "file.path": [
386
381
  "/home/bob/file.txt",
387
- "c:\\users\\bob\\desktop\\test.txt",
388
382
  "c:/users/bob smith/desktop/file.txt:alt.exe",
389
- "c:\\users\\bob smith\\desktop\\file.txt:alt.exe",
383
+ "c:\\users\\bob\\desktop\\test.txt",
390
384
  "file.txt:alt.exe",
391
- "c:\\users\\bob smith\\desktop\\file.txt",
392
- "c:\\users\\bob smith\\desktop\\file2.txt:alt.exe"
385
+ "c:\\users\\bob smith\\desktop\\file.txt:alt.exe",
386
+ "c:\\users\\bob smith\\desktop\\file2.txt:alt.exe",
387
+ "c:\\users\\bob smith\\desktop\\file.txt"
393
388
  ],
394
389
  "file.name": [
395
390
  "/home/bob/file.txt",
396
- "c:\\users\\bob\\desktop\\test.txt",
397
391
  "c:/users/bob smith/desktop/file.txt:alt.exe",
398
- "c:\\users\\bob smith\\desktop\\file.txt:alt.exe",
392
+ "c:\\users\\bob\\desktop\\test.txt",
399
393
  "file.txt:alt.exe",
400
- "c:\\users\\bob smith\\desktop\\file.txt",
401
- "c:\\users\\bob smith\\desktop\\file2.txt:alt.exe"
394
+ "c:\\users\\bob smith\\desktop\\file.txt:alt.exe",
395
+ "c:\\users\\bob smith\\desktop\\file2.txt:alt.exe",
396
+ "c:\\users\\bob smith\\desktop\\file.txt"
402
397
  ],
403
398
  "dll.hash.md5": [
404
399
  "6cd3556deb0da54bca060b4c39479839"
@@ -445,9 +440,40 @@ print(json.dumps(m, indent=4))
445
440
  8443
446
441
  ]
447
442
  }
443
+ ========================== output to Sqlite database ===========================
444
+ exported 34 indicators to out.db
445
+ ('bad.username@subdomain.bad.local', 'email')
446
+ ('bob@email.local', 'email')
447
+ ('n--nhk-u63b1cko2lyc6jrwxgom6k.com', 'domain')
448
+ ('website.local', 'domain')
449
+ ('email.local', 'domain')
450
+ ('securewebsite.local', 'domain')
451
+ ('bad.local', 'domain')
452
+ ('subdomain.bad.local', 'domain')
453
+ ('anotherwebsite.local', 'domain')
454
+ ('8.8.8.8', 'ipv4')
455
+ ('192.168.20.20', 'ipv4')
456
+ ('192.168.1.1', 'ipv4')
457
+ ('1.2.3.4', 'ipv4')
458
+ ('9443', 'port')
459
+ ('8443', 'port')
460
+ ('username:password', 'credentials')
461
+ ('192.168.1.0/24', 'ipv4_network')
462
+ ('8.8.8.0/24', 'ipv4_network')
463
+ ('fe80::', 'ipv6')
464
+ ('/home/bob/file.txt', 'file_path_linux')
465
+ ('file.txt', 'file')
466
+ ('rc:\\users\\bob smith\\desktop\\file.txt:alt.exe', 'file')
467
+ ('aaaaaaaaaaaaaaaa', 'file')
468
+ ('bob documents.pdf', 'file')
469
+ ('not-an-ioc', 'file')
470
+ ('c:\\users\\bob smith\\desktop\\file.txt', 'file_path_windows')
471
+ ('c:/users/bob smith/desktop/file.txt:alt.exe', 'file_path_windows')
472
+ ('file.txt:alt.exe', 'file_path_windows')
473
+ ('c:\\users\\bob smith\\desktop\\file.txt:alt.exe', 'file_path_windows')
474
+ ('c:\\users\\bob smith\\desktop\\file2.txt:alt.exe', 'file_path_windows')
475
+ ('c:\\users\\bob\\desktop\\test.txt', 'file_path_windows')
476
+ ('6cd3556deb0da54bca060b4c39479839', 'md5')
477
+ ('315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3', 'sha256')
478
+ ('c1527cd893c124773d811911970c8fe6e857d6df5dc9226bd8a160614c0cd963a4ddea2b94bb7d36021ef9d865d5cea294a82dd49a0bb269f51f6e7a57f79421', 'sha512')
448
479
  ```
449
-
450
- ## Known Issues
451
-
452
- - need to streamline and optimize code
453
- - remove both regex statements created by AI