cyvest 0.1.0__py3-none-any.whl โ 5.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cyvest/__init__.py +48 -38
- cyvest/cli.py +487 -0
- cyvest/compare.py +318 -0
- cyvest/cyvest.py +1431 -0
- cyvest/investigation.py +1682 -0
- cyvest/io_rich.py +1153 -0
- cyvest/io_schema.py +35 -0
- cyvest/io_serialization.py +465 -0
- cyvest/io_visualization.py +358 -0
- cyvest/keys.py +237 -0
- cyvest/level_score_rules.py +78 -0
- cyvest/levels.py +175 -0
- cyvest/model.py +595 -0
- cyvest/model_enums.py +69 -0
- cyvest/model_schema.py +164 -0
- cyvest/proxies.py +595 -0
- cyvest/score.py +473 -0
- cyvest/shared.py +508 -0
- cyvest/stats.py +291 -0
- cyvest/ulid.py +36 -0
- cyvest-5.1.3.dist-info/METADATA +632 -0
- cyvest-5.1.3.dist-info/RECORD +24 -0
- {cyvest-0.1.0.dist-info โ cyvest-5.1.3.dist-info}/WHEEL +1 -2
- cyvest-5.1.3.dist-info/entry_points.txt +3 -0
- cyvest/builder.py +0 -182
- cyvest/check_tree.py +0 -117
- cyvest/models.py +0 -785
- cyvest/observable_registry.py +0 -69
- cyvest/report_render.py +0 -306
- cyvest/report_serialization.py +0 -237
- cyvest/visitors.py +0 -332
- cyvest-0.1.0.dist-info/METADATA +0 -110
- cyvest-0.1.0.dist-info/RECORD +0 -13
- cyvest-0.1.0.dist-info/licenses/LICENSE +0 -21
- cyvest-0.1.0.dist-info/top_level.txt +0 -1
|
@@ -0,0 +1,632 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: cyvest
|
|
3
|
+
Version: 5.1.3
|
|
4
|
+
Summary: Cybersecurity investigation model
|
|
5
|
+
Keywords: cybersecurity,investigation,threat-intel,security-analysis
|
|
6
|
+
Author: PakitoSec
|
|
7
|
+
Author-email: PakitoSec <jeromep83@gmail.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Security
|
|
17
|
+
Requires-Dist: click>=8
|
|
18
|
+
Requires-Dist: logurich[click]>=0.6.0
|
|
19
|
+
Requires-Dist: pydantic>=2.12.5
|
|
20
|
+
Requires-Dist: rich>=13
|
|
21
|
+
Requires-Dist: typing-extensions>=4.15
|
|
22
|
+
Requires-Dist: pyvis>=0.3.2 ; extra == 'visualization'
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Project-URL: Homepage, https://github.com/PakitoSec/cyvest
|
|
25
|
+
Project-URL: Repository, https://github.com/PakitoSec/cyvest
|
|
26
|
+
Project-URL: Issues, https://github.com/PakitoSec/cyvest/issues
|
|
27
|
+
Provides-Extra: visualization
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# Cyvest - Cybersecurity Investigation Framework
|
|
31
|
+
|
|
32
|
+
[](https://www.python.org/downloads/)
|
|
33
|
+
[](https://opensource.org/licenses/MIT)
|
|
34
|
+
|
|
35
|
+
**Cyvest** is a Python framework for building, analyzing, and structuring cybersecurity investigations programmatically. It provides automatic scoring, level calculation, relationship tracking, and rich reporting capabilities.
|
|
36
|
+
|
|
37
|
+
## Features
|
|
38
|
+
|
|
39
|
+
- ๐ **Structured Investigation Modeling**: Model investigations with observables, checks, threat intelligence, and enrichments
|
|
40
|
+
- ๐ **Automatic Scoring**: Dynamic score calculation and propagation through investigation hierarchy
|
|
41
|
+
- ๐ฏ **Level Classification**: Automatic security level assignment (TRUSTED, INFO, SAFE, NOTABLE, SUSPICIOUS, MALICIOUS)
|
|
42
|
+
- ๐ **Relationship Tracking**: Lightweight relationship modeling between observables
|
|
43
|
+
- ๐ท๏ธ **Typed Helpers**: Built-in enums for observable types and relationships with autocomplete
|
|
44
|
+
- ๐ **Real-time Statistics**: Live metrics and aggregations throughout the investigation
|
|
45
|
+
- ๐ **Investigation Merging**: Combine investigations from multiple threads or processes
|
|
46
|
+
- ๐งต **Multi-Threading Support**: Advanced thread-safe shared context available via `cyvest.shared`
|
|
47
|
+
- ๐พ **Multiple Export Formats**: JSON and Markdown output for reporting and LLM consumption
|
|
48
|
+
- ๐จ **Rich Console Output**: Beautiful terminal displays with the Rich library
|
|
49
|
+
- ๐งฉ **Fluent helpers**: Convenient API with method chaining for rapid development
|
|
50
|
+
- ๐ฌ **Investigation Comparison**: Compare investigations with tolerance rules and visual diff output
|
|
51
|
+
|
|
52
|
+
## Installation
|
|
53
|
+
|
|
54
|
+
### Using uv (recommended)
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
# Install uv if not already installed
|
|
58
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
59
|
+
|
|
60
|
+
# Clone the repository
|
|
61
|
+
git clone https://github.com/PakitoSec/cyvest.git
|
|
62
|
+
cd cyvest
|
|
63
|
+
|
|
64
|
+
# Install dependencies
|
|
65
|
+
uv sync
|
|
66
|
+
|
|
67
|
+
# Install in development mode
|
|
68
|
+
uv pip install -e .
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### Using pip
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install -e .
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
> Install the optional visualization extra with\
|
|
78
|
+
> `pip install "cyvest[visualization]"` (or `uv pip install -e ".[visualization]"`).
|
|
79
|
+
|
|
80
|
+
## Quick Start
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from decimal import Decimal
|
|
84
|
+
from cyvest import Cyvest
|
|
85
|
+
|
|
86
|
+
# Create an investigation (root_data becomes the root observable extra)
|
|
87
|
+
cv = Cyvest(root_data={"type": "email"})
|
|
88
|
+
|
|
89
|
+
# For deterministic reports (enables diffing between runs), pass a custom investigation_id:
|
|
90
|
+
# cv = Cyvest(root_data={"type": "email"}, investigation_id="email-analysis-v1")
|
|
91
|
+
|
|
92
|
+
# Create observables
|
|
93
|
+
url = (
|
|
94
|
+
cv.observable(cv.OBS.URL, "https://phishing-site.com", internal=False)
|
|
95
|
+
.with_ti("virustotal", score=Decimal("8.5"), level=cv.LVL.MALICIOUS)
|
|
96
|
+
.relate_to(cv.root(), cv.REL.RELATED_TO)
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Create checks
|
|
100
|
+
check = cv.check("url_analysis", "email_body", "Analyze suspicious URL")
|
|
101
|
+
check.link_observable(url)
|
|
102
|
+
check.with_score(Decimal("8.5"), "Malicious URL detected")
|
|
103
|
+
|
|
104
|
+
# Display results
|
|
105
|
+
print(f"Global Score: {cv.get_global_score()}")
|
|
106
|
+
print(f"Global Level: {cv.get_global_level()}")
|
|
107
|
+
|
|
108
|
+
# Export
|
|
109
|
+
cv.io_save_json("investigation.json")
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Model Proxies
|
|
113
|
+
|
|
114
|
+
Cyvest only exposes immutable model proxies. Helpers like `observable_create`, `check_create`, and the
|
|
115
|
+
fluent `cv.observable()`/`cv.check()` convenience methods return `ObservableProxy`, `CheckProxy`, `TagProxy`, etc.
|
|
116
|
+
These proxies reflect the live investigation state but raise `AttributeError` if you try to assign to their attributes.
|
|
117
|
+
All mutations are routed through the Investigation layer, so use the facade helpers (`cv.observable_set_level`,
|
|
118
|
+
`cv.check_update_score`, `cv.observable_add_threat_intel`) or the built-in fluent methods on the proxies themselves
|
|
119
|
+
(`with_ti`, `relate_to`, `link_observable`, `with_score`, `set_level`, โฆ) so the score engine and audit log stay consistent.
|
|
120
|
+
|
|
121
|
+
Mutation helpers that reference existing objects (for example, `cv.observable_add_relationship`,
|
|
122
|
+
`cv.check_link_observable`, `cv.tag_add_check`) raise `KeyError` when a key is missing.
|
|
123
|
+
|
|
124
|
+
Safe metadata fields like `comment`, `extra`, or `internal` can be updated through the proxies without breaking score
|
|
125
|
+
consistency. Use `set_level()` to update the level without changing the score:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
url_obs.update_metadata(comment="triaged", internal=False, extra={"ticket": "INC-4242"})
|
|
129
|
+
check.update_metadata(description="New scope", extra={"playbook": "url-analysis"})
|
|
130
|
+
check.set_level(cv.LVL.SAFE, reason="Verified clean")
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Dictionary fields merge by default; pass `merge_extra=False` (or `merge_data=False` for enrichments) to overwrite them.
|
|
134
|
+
|
|
135
|
+
### Threat Intel Drafts
|
|
136
|
+
|
|
137
|
+
When the observable is unknown yet, create a draft and attach it later:
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
draft = cv.threat_intel_draft("vt", score=Decimal("4.2"), comment="Initial lookup")
|
|
141
|
+
obs = cv.observable(cv.OBS.DOMAIN, "example.com")
|
|
142
|
+
obs.with_ti_draft(draft)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Drafts are plain `ThreatIntel` objects with no `observable_key` yet; attaching generates the key.
|
|
146
|
+
|
|
147
|
+
## Core Concepts
|
|
148
|
+
|
|
149
|
+
### Observables
|
|
150
|
+
|
|
151
|
+
Observables represent cyber artifacts (URLs, IPs, domains, hashes, files, etc.).
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
from cyvest import Cyvest
|
|
155
|
+
|
|
156
|
+
cv = Cyvest()
|
|
157
|
+
|
|
158
|
+
url_obs = cv.observable_create(cv.OBS.URL, "https://malicious.com", internal=False)
|
|
159
|
+
|
|
160
|
+
ip_obs = cv.observable_create(cv.OBS.IPV4, "192.0.2.1", internal=False)
|
|
161
|
+
|
|
162
|
+
cv.observable_add_relationship(
|
|
163
|
+
url_obs, # Can pass ObservableProxy directly
|
|
164
|
+
ip_obs, # Or use .key for string keys
|
|
165
|
+
cv.REL.RELATED_TO,
|
|
166
|
+
cv.DIR.BIDIRECTIONAL,
|
|
167
|
+
)
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Cyvest exposes enums for observable types and relationships via the facade (`cv.OBS`, `cv.REL`, `cv.DIR`)
|
|
171
|
+
so IDEs can autocomplete the official vocabulary without extra imports.
|
|
172
|
+
|
|
173
|
+
### Checks
|
|
174
|
+
|
|
175
|
+
Checks represent verification steps in your investigation:
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
check = cv.check_create(
|
|
179
|
+
check_id="malware_detection",
|
|
180
|
+
scope="endpoint",
|
|
181
|
+
description="Verify file hash against threat intel",
|
|
182
|
+
score=Decimal("8.0"),
|
|
183
|
+
level=cv.LVL.MALICIOUS
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
# Link observables to checks
|
|
187
|
+
cv.check_link_observable(check.key, file_hash_obs.key)
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### Threat Intelligence
|
|
191
|
+
|
|
192
|
+
Threat intelligence provides verdicts from external sources:
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
cv.observable_add_threat_intel(
|
|
196
|
+
observable.key,
|
|
197
|
+
source="virustotal",
|
|
198
|
+
score=Decimal("7.5"),
|
|
199
|
+
level=cv.LVL.SUSPICIOUS,
|
|
200
|
+
comment="15/70 vendors flagged as malicious",
|
|
201
|
+
taxonomies=[cv.taxonomy(level=cv.LVL.MALICIOUS, name="scan", value="trojan")]
|
|
202
|
+
)
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Taxonomies are unique by name per threat intel entry. Use the fluent helpers to add or remove them:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
ti = cv.observable_add_threat_intel(observable.key, source="vt", score=Decimal("7.5"))
|
|
209
|
+
ti.add_taxonomy(level=cv.LVL.SUSPICIOUS, name="confidence", value="medium")
|
|
210
|
+
ti.remove_taxonomy("confidence")
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### Tags
|
|
214
|
+
|
|
215
|
+
Tags organize checks with automatic hierarchy based on `:` delimiter:
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
# Simple: pass tag names directly (auto-creates tags)
|
|
219
|
+
check = cv.check("beacon_detection", "Detect C2 beacons")
|
|
220
|
+
check.tagged("network", "c2:detection", "suspicious")
|
|
221
|
+
|
|
222
|
+
# With description: create tag first, then reference it
|
|
223
|
+
tag = cv.tag("network:c2:detection", "C2 Detection Checks")
|
|
224
|
+
check.tagged(tag)
|
|
225
|
+
|
|
226
|
+
# Query hierarchy
|
|
227
|
+
children = cv.tag_get_children("network") # ["network:c2"]
|
|
228
|
+
descendants = cv.tag_get_descendants("network") # ["network:c2", "network:c2:detection"]
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
### Lookup Helpers
|
|
232
|
+
|
|
233
|
+
Use facade getters with either key strings or component parameters:
|
|
234
|
+
|
|
235
|
+
```python
|
|
236
|
+
url_obs = cv.observable_create(cv.OBS.URL, "https://malicious.com")
|
|
237
|
+
same_url = cv.observable_get(cv.OBS.URL, "https://malicious.com")
|
|
238
|
+
same_url_by_key = cv.observable_get(url_obs.key)
|
|
239
|
+
|
|
240
|
+
check = cv.check_create("malware_detection", "endpoint", "Verify file hash")
|
|
241
|
+
same_check = cv.check_get("malware_detection", "endpoint")
|
|
242
|
+
same_check_by_key = cv.check_get(check.key)
|
|
243
|
+
|
|
244
|
+
tag = cv.tag_create("network:analysis")
|
|
245
|
+
same_tag = cv.tag_get("network:analysis")
|
|
246
|
+
same_tag_by_key = cv.tag_get(tag.key)
|
|
247
|
+
|
|
248
|
+
enrichment = cv.enrichment_create("whois", {"registrar": "Example Inc"})
|
|
249
|
+
same_enrichment = cv.enrichment_get("whois")
|
|
250
|
+
same_enrichment_by_key = cv.enrichment_get(enrichment.key)
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
Low-level `Investigation` getters accept keys only; use the facade for component-based lookups.
|
|
254
|
+
|
|
255
|
+
### Multi-Threaded Investigations
|
|
256
|
+
|
|
257
|
+
**Advanced Feature**: Use `Cyvest.shared_context()` (or `SharedInvestigationContext` from `cyvest.shared`) for safe parallel task execution with automatic observable sharing:
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
from cyvest import Cyvest
|
|
261
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
262
|
+
|
|
263
|
+
def email_analysis(shared_context):
|
|
264
|
+
# create_cyvest() yields a task-local Cyvest that auto-merges on context exit
|
|
265
|
+
with shared_context.create_cyvest() as cy:
|
|
266
|
+
data = cy.root().extra
|
|
267
|
+
cy.observable(cy.OBS.DOMAIN, data.get("domain"))
|
|
268
|
+
|
|
269
|
+
# Create shared context
|
|
270
|
+
main_cy = Cyvest(root_data=email_data, root_type=Cyvest.OBS.ARTIFACT)
|
|
271
|
+
shared = main_cy.shared_context()
|
|
272
|
+
|
|
273
|
+
# Run tasks in parallel - they can reference each other's observables
|
|
274
|
+
with ThreadPoolExecutor(max_workers=4) as executor:
|
|
275
|
+
futures = [executor.submit(email_analysis, shared) for _ in tasks]
|
|
276
|
+
for future in as_completed(futures):
|
|
277
|
+
future.result() # Auto-reconciled
|
|
278
|
+
|
|
279
|
+
# Get merged investigation (same object passed to shared_context)
|
|
280
|
+
final_cy = main_cy
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
See `examples/04_email.py` for a complete multi-threaded investigation example.
|
|
284
|
+
|
|
285
|
+
### Scoring & Levels
|
|
286
|
+
|
|
287
|
+
Scores and levels are automatically calculated and propagated:
|
|
288
|
+
|
|
289
|
+
- **Threat Intel โ Observable**: Observable score = **max** of all threat intel scores (not sum)
|
|
290
|
+
- **Observable Hierarchy**: Parent observable scores include child observable scores based on relationship direction:
|
|
291
|
+
- **OUTBOUND relationships**: target scores propagate to source (source is parent)
|
|
292
|
+
- **INBOUND relationships**: source scores propagate to target (target is parent)
|
|
293
|
+
- **BIDIRECTIONAL relationships**: no hierarchical propagation
|
|
294
|
+
- **Observable โ Check (provenance-aware)**: Check score/level only considers observables reachable through *effective* links (`observable_links`)
|
|
295
|
+
- A link is effective when `propagation_mode="GLOBAL"` or when the check's `origin_investigation_id` matches the current investigation id
|
|
296
|
+
- **Check โ Global**: All check scores sum to global investigation score
|
|
297
|
+
|
|
298
|
+
Observable score aggregation is configurable via `score_mode_obs`:
|
|
299
|
+
|
|
300
|
+
```python
|
|
301
|
+
from cyvest import Cyvest
|
|
302
|
+
from cyvest.score import ScoreMode
|
|
303
|
+
|
|
304
|
+
cv = Cyvest(score_mode_obs=ScoreMode.MAX) # default
|
|
305
|
+
cv = Cyvest(score_mode_obs=ScoreMode.SUM) # accumulative children
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
**Provenance model**
|
|
309
|
+
|
|
310
|
+
- `Investigation.investigation_id` is a stable ULID included in exports.
|
|
311
|
+
- Checks keep a *canonical origin* (`origin_investigation_id`) for LOCAL_ONLY propagation; it is compared against the current investigation id.
|
|
312
|
+
|
|
313
|
+
**Audit log**
|
|
314
|
+
|
|
315
|
+
- All meaningful changes (including score/level changes) are recorded in the investigation-level audit log.
|
|
316
|
+
- Per-object histories are not stored; use `cv.investigation_get_audit_log()` to review changes.
|
|
317
|
+
- For compact, deterministic JSON output (useful for testing/diffing), exclude the audit log:
|
|
318
|
+
```python
|
|
319
|
+
cv.io_save_json("output.json", include_audit_log=False) # audit_log: null
|
|
320
|
+
cv.io_to_invest(include_audit_log=False) # schema.audit_log is None
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
To force cross-investigation propagation for a specific link, use a GLOBAL link:
|
|
324
|
+
|
|
325
|
+
```python
|
|
326
|
+
cv.check_link_observable(check.key, observable.key, propagation_mode="GLOBAL")
|
|
327
|
+
# or fluent:
|
|
328
|
+
cv.check("id", "scope", "desc").link_observable(observable, propagation_mode="GLOBAL")
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
Score to Level mapping:
|
|
332
|
+
|
|
333
|
+
- `< 0.0` โ TRUSTED
|
|
334
|
+
- `== 0.0` โ INFO
|
|
335
|
+
- `< 3.0` โ NOTABLE
|
|
336
|
+
- `< 5.0` โ SUSPICIOUS
|
|
337
|
+
- `>= 5.0` โ MALICIOUS
|
|
338
|
+
|
|
339
|
+
**SAFE Level Protection:**
|
|
340
|
+
|
|
341
|
+
The SAFE level has special protection for trusted/whitelisted observables:
|
|
342
|
+
|
|
343
|
+
```python
|
|
344
|
+
# Mark a known-good domain as SAFE
|
|
345
|
+
trusted = cv.observable_create(
|
|
346
|
+
cv.OBS.DOMAIN,
|
|
347
|
+
"trusted.example.com",
|
|
348
|
+
level=cv.LVL.SAFE
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
# Adding low-score threat intel won't downgrade to TRUSTED or INFO
|
|
352
|
+
cv.observable_add_threat_intel(trusted.key, "source1", score=Decimal("0"))
|
|
353
|
+
# Level stays SAFE, score updates to 0
|
|
354
|
+
|
|
355
|
+
# But high-score threat intel can still upgrade to MALICIOUS if warranted
|
|
356
|
+
cv.observable_add_threat_intel(trusted.key, "source2", score=Decimal("6.0"))
|
|
357
|
+
# Level upgrades to MALICIOUS, score updates to 6.0
|
|
358
|
+
|
|
359
|
+
# Threat intel with SAFE level can also mark observables as SAFE
|
|
360
|
+
uncertain = cv.observable_create(cv.OBS.DOMAIN, "example.com")
|
|
361
|
+
cv.observable_add_threat_intel(
|
|
362
|
+
uncertain.key,
|
|
363
|
+
"whitelist_service",
|
|
364
|
+
score=Decimal("0"),
|
|
365
|
+
level=cv.LVL.SAFE
|
|
366
|
+
)
|
|
367
|
+
# Observable upgraded to SAFE level with automatic downgrade protection
|
|
368
|
+
```
|
|
369
|
+
|
|
370
|
+
SAFE observables:
|
|
371
|
+
- Cannot be downgraded to lower levels (NONE, TRUSTED, INFO)
|
|
372
|
+
- Can be upgraded to higher levels (NOTABLE, SUSPICIOUS, MALICIOUS)
|
|
373
|
+
- Score values still update based on threat intelligence
|
|
374
|
+
- Protection is preserved during investigation merges
|
|
375
|
+
- Can be marked SAFE by threat intel sources (e.g., whitelists, reputation databases)
|
|
376
|
+
|
|
377
|
+
SAFE checks:
|
|
378
|
+
- Automatically inherit SAFE level when linked to SAFE observables (if all other observables are โค SAFE)
|
|
379
|
+
- Can still upgrade to higher levels when NOTABLE/SUSPICIOUS/MALICIOUS observables are linked
|
|
380
|
+
|
|
381
|
+
**Root Observable Barrier:**
|
|
382
|
+
|
|
383
|
+
The root observable (the investigation's entry point with `value="root"`) acts as a special barrier to prevent cross-contamination:
|
|
384
|
+
Its key is derived from type + value (e.g. `obs:file:root` or `obs:artifact:root`).
|
|
385
|
+
|
|
386
|
+
**Barrier as Child** - When root appears as a child of other observables, it is **skipped** in their score calculations.
|
|
387
|
+
|
|
388
|
+
**Barrier as Parent** - Root's propagation is asymmetric:
|
|
389
|
+
- Root **CAN** be updated when children change (aggregates child scores)
|
|
390
|
+
- Root **does NOT** propagate upward beyond itself (stops recursive propagation)
|
|
391
|
+
- Root **DOES** propagate to checks normally
|
|
392
|
+
|
|
393
|
+
This design enables flexible investigation structures while preventing unintended score contamination.
|
|
394
|
+
|
|
395
|
+
### Comparing Investigations
|
|
396
|
+
|
|
397
|
+
Compare two investigations to identify differences in checks, observables, and threat intelligence:
|
|
398
|
+
|
|
399
|
+
```python
|
|
400
|
+
from decimal import Decimal
|
|
401
|
+
from cyvest import Cyvest, ExpectedResult, Level, compare_investigations
|
|
402
|
+
from cyvest.io_rich import display_diff
|
|
403
|
+
|
|
404
|
+
# Create expected and actual investigations
|
|
405
|
+
expected = Cyvest(investigation_name="expected")
|
|
406
|
+
expected.check_create("domain-check", "Verify domain", score=Decimal("1.0"))
|
|
407
|
+
|
|
408
|
+
actual = Cyvest(investigation_name="actual")
|
|
409
|
+
actual.check_create("domain-check", "Verify domain", score=Decimal("2.0"))
|
|
410
|
+
actual.check_create("new-check", "New detection", score=Decimal("1.5"))
|
|
411
|
+
|
|
412
|
+
# Compare investigations
|
|
413
|
+
diffs = compare_investigations(actual, expected)
|
|
414
|
+
# diffs contains:
|
|
415
|
+
# - MISMATCH for domain-check (score changed 1.0 -> 2.0)
|
|
416
|
+
# - ADDED for new-check
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
**Tolerance Rules**
|
|
420
|
+
|
|
421
|
+
Use `result_expected` rules to define acceptable score variations:
|
|
422
|
+
|
|
423
|
+
```python
|
|
424
|
+
# Define tolerance rules
|
|
425
|
+
rules = [
|
|
426
|
+
# Accept any score >= 1.0 for this check
|
|
427
|
+
ExpectedResult(check_name="domain-check", score=">= 1.0"),
|
|
428
|
+
# Accept any score < 3.0 for roger-ai
|
|
429
|
+
ExpectedResult(key="chk:roger-ai", level=Level.SUSPICIOUS, score="< 3.0"),
|
|
430
|
+
]
|
|
431
|
+
|
|
432
|
+
# Compare with tolerance - checks satisfying rules are not flagged as diffs
|
|
433
|
+
diffs = compare_investigations(actual, expected, result_expected=rules)
|
|
434
|
+
```
|
|
435
|
+
|
|
436
|
+
Supported operators: `>=`, `<=`, `>`, `<`, `==`, `!=`
|
|
437
|
+
|
|
438
|
+
**Visual Diff Display**
|
|
439
|
+
|
|
440
|
+
Display differences in a rich table format:
|
|
441
|
+
|
|
442
|
+
```python
|
|
443
|
+
from cyvest.io_rich import display_diff
|
|
444
|
+
from logurich import logger
|
|
445
|
+
|
|
446
|
+
# Display diff table with tree structure showing observables and threat intel
|
|
447
|
+
display_diff(diffs, lambda r: logger.rich("INFO", r), title="Investigation Diff")
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
Output:
|
|
451
|
+
```
|
|
452
|
+
โญโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโฌโโโโโโโโโโโโโโโโโโโโโฌโโโโโโโโโโโโโโโโโโฌโโโโโโโโโฎ
|
|
453
|
+
โ Key โ Expected โ Actual โ Status โ
|
|
454
|
+
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโผโโโโโโโโโค
|
|
455
|
+
โ chk:new-check โ - โ NOTABLE 1.50 โ + โ
|
|
456
|
+
โ โโโ domain: example.com โ - โ INFO 0.00 โ โ
|
|
457
|
+
โ โโโ VirusTotal โ - โ INFO 0.00 โ โ
|
|
458
|
+
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโผโโโโโโโโโค
|
|
459
|
+
โ chk:domain-check โ NOTABLE 1.00 โ NOTABLE 2.00 โ โ โ
|
|
460
|
+
โฐโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโดโโโโโโโโโฏ
|
|
461
|
+
```
|
|
462
|
+
|
|
463
|
+
Status symbols: `+` (added), `-` (removed), `โ` (mismatch)
|
|
464
|
+
|
|
465
|
+
**Convenience Methods**
|
|
466
|
+
|
|
467
|
+
Use methods directly on Cyvest objects:
|
|
468
|
+
|
|
469
|
+
```python
|
|
470
|
+
# Compare and get diff items
|
|
471
|
+
diffs = actual.compare(expected=expected, result_expected=rules)
|
|
472
|
+
|
|
473
|
+
# Compare and display in one call
|
|
474
|
+
actual.display_diff(expected=expected, title="My Investigation Diff")
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
## Examples
|
|
478
|
+
|
|
479
|
+
See the `examples/` directory for complete examples:
|
|
480
|
+
|
|
481
|
+
- **01_email_basic.py**: Basic email phishing investigation
|
|
482
|
+
- **02_urls_and_ips.py**: Network investigation with URLs and IPs
|
|
483
|
+
- **03_merge_demo.py**: Multi-process investigation merging
|
|
484
|
+
- **04_email.py**: Multi-threaded investigation with SharedInvestigationContext
|
|
485
|
+
- **05_visualization.py**: Interactive HTML visualization showcasing scores, levels, and relationship flows
|
|
486
|
+
- **06_compare_investigations.py**: Compare investigations with tolerance rules and visual diff output
|
|
487
|
+
|
|
488
|
+
Run an example:
|
|
489
|
+
|
|
490
|
+
```bash
|
|
491
|
+
python examples/01_email_basic.py
|
|
492
|
+
python examples/04_email.py
|
|
493
|
+
python examples/05_visualization.py
|
|
494
|
+
```
|
|
495
|
+
|
|
496
|
+
## CLI Usage
|
|
497
|
+
|
|
498
|
+
Cyvest includes a command-line interface for working with investigation files:
|
|
499
|
+
|
|
500
|
+
```bash
|
|
501
|
+
# Display investigation
|
|
502
|
+
cyvest show investigation.json --graph
|
|
503
|
+
|
|
504
|
+
# Show statistics
|
|
505
|
+
cyvest stats investigation.json --detailed
|
|
506
|
+
|
|
507
|
+
# Export to markdown
|
|
508
|
+
cyvest export investigation.json -o report.md -f markdown
|
|
509
|
+
|
|
510
|
+
# Merge investigations with automatic deduplication
|
|
511
|
+
cyvest merge inv1.json inv2.json inv3.json -o merged.json
|
|
512
|
+
|
|
513
|
+
# Merge with statistics display
|
|
514
|
+
cyvest merge inv1.json inv2.json -o merged.json --stats
|
|
515
|
+
|
|
516
|
+
# Merge and display rich summary
|
|
517
|
+
cyvest merge inv1.json inv2.json -o merged.json -f rich --stats
|
|
518
|
+
|
|
519
|
+
# Generate an interactive visualization (requires visualization extra)
|
|
520
|
+
cyvest visualize investigation.json --min-level SUSPICIOUS --group-by-type
|
|
521
|
+
|
|
522
|
+
# Output the JSON Schema describing serialized investigations and generate types
|
|
523
|
+
uv run cyvest schema -o ./schema/cyvest.schema.json && pnpm -C js/packages/cyvest-js run generate:types
|
|
524
|
+
```
|
|
525
|
+
|
|
526
|
+
## Development
|
|
527
|
+
|
|
528
|
+
### Setup Development Environment
|
|
529
|
+
|
|
530
|
+
```bash
|
|
531
|
+
# Install development dependencies
|
|
532
|
+
uv sync --all-extras
|
|
533
|
+
|
|
534
|
+
# Run tests
|
|
535
|
+
pytest
|
|
536
|
+
|
|
537
|
+
# Run tests with coverage
|
|
538
|
+
pytest --cov=cyvest --cov-report=html
|
|
539
|
+
|
|
540
|
+
# Format code
|
|
541
|
+
ruff format .
|
|
542
|
+
|
|
543
|
+
# Lint code
|
|
544
|
+
ruff check .
|
|
545
|
+
```
|
|
546
|
+
|
|
547
|
+
### Running Tests
|
|
548
|
+
|
|
549
|
+
```bash
|
|
550
|
+
# Run all tests
|
|
551
|
+
pytest
|
|
552
|
+
|
|
553
|
+
# Run specific test file
|
|
554
|
+
pytest tests/test_score.py
|
|
555
|
+
|
|
556
|
+
# Run with verbose output
|
|
557
|
+
pytest -v
|
|
558
|
+
|
|
559
|
+
# Run with coverage
|
|
560
|
+
pytest --cov=cyvest
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
## Documentation
|
|
564
|
+
|
|
565
|
+
Build the documentation with MkDocs:
|
|
566
|
+
|
|
567
|
+
```bash
|
|
568
|
+
# Install docs dependencies
|
|
569
|
+
uv sync --all-extras
|
|
570
|
+
|
|
571
|
+
# Serve documentation locally
|
|
572
|
+
mkdocs serve
|
|
573
|
+
|
|
574
|
+
# Build documentation
|
|
575
|
+
mkdocs build
|
|
576
|
+
```
|
|
577
|
+
|
|
578
|
+
## JavaScript packages
|
|
579
|
+
|
|
580
|
+
The repo includes a PNPM workspace under `js/` with three packages:
|
|
581
|
+
|
|
582
|
+
- `@cyvest/cyvest-js`: TypeScript types, schema validation, and helpers for Cyvest investigations.
|
|
583
|
+
- `@cyvest/cyvest-vis`: React components for graph visualization (depends on `@cyvest/cyvest-js`).
|
|
584
|
+
- `@cyvest/cyvest-app`: Vite demo that bundles the JS packages with sample investigations.
|
|
585
|
+
|
|
586
|
+
The JS packages track the generated schema; serialized investigations should include fields like
|
|
587
|
+
`investigation_id`, `investigation_name`, `audit_log`, `score_display`, `check_links`, and
|
|
588
|
+
`observable_links`. The investigation start time is recorded as an `INVESTIGATION_STARTED` event
|
|
589
|
+
in the `audit_log`.
|
|
590
|
+
|
|
591
|
+
See `docs/js-packages.md` for workspace commands and usage snippets.
|
|
592
|
+
|
|
593
|
+
## Contributing
|
|
594
|
+
|
|
595
|
+
Contributions are welcome! Please:
|
|
596
|
+
|
|
597
|
+
1. Fork the repository
|
|
598
|
+
2. Create a feature branch
|
|
599
|
+
3. Make your changes with tests
|
|
600
|
+
4. Run the test suite
|
|
601
|
+
5. Submit a pull request
|
|
602
|
+
|
|
603
|
+
## License
|
|
604
|
+
|
|
605
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
606
|
+
|
|
607
|
+
## Use Cases
|
|
608
|
+
|
|
609
|
+
Cyvest is designed for:
|
|
610
|
+
|
|
611
|
+
- **Security Operations Centers (SOCs)**: Automate investigation workflows
|
|
612
|
+
- **Incident Response**: Structure and document incident investigations
|
|
613
|
+
- **Threat Hunting**: Build repeatable hunting methodologies
|
|
614
|
+
- **Malware Analysis**: Track relationships between artifacts
|
|
615
|
+
- **Phishing Analysis**: Analyze emails and linked resources
|
|
616
|
+
- **Integration**: Combine results from multiple security tools
|
|
617
|
+
- **Regression Testing**: Compare investigation outputs across rule or detection updates
|
|
618
|
+
|
|
619
|
+
## Architecture Highlights
|
|
620
|
+
|
|
621
|
+
- **Concurrency**: Advanced `SharedInvestigationContext` (via `cyvest.shared`) enables safe parallel task execution
|
|
622
|
+
- **Deterministic Keys**: Same objects always generate same keys for merging
|
|
623
|
+
- **Deterministic IDs**: Optional `investigation_id` parameter for reproducible reports and diffing
|
|
624
|
+
- **Score Propagation**: Automatic hierarchical score calculation
|
|
625
|
+
- **Flexible Export**: JSON for storage, Markdown for LLM analysis
|
|
626
|
+
- **Audit Trail**: Score change history for debugging
|
|
627
|
+
- **Investigation Comparison**: Compare investigations with tolerance rules for regression testing
|
|
628
|
+
|
|
629
|
+
## Future Enhancements
|
|
630
|
+
|
|
631
|
+
- Database persistence layer
|
|
632
|
+
- Additional export formats (PDF, HTML)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
cyvest/__init__.py,sha256=2CDmTJB7H39v4Wiz-UfTYXqVBLzveAuLMVKmAtzpRRQ,1383
|
|
2
|
+
cyvest/cli.py,sha256=M-VgZzBp4lvBVxzSmTDPyPI9WzYdoenCYT3XTiD3LGs,16262
|
|
3
|
+
cyvest/compare.py,sha256=K9j3dLPjCpTfBcfDlnYZ_pndA97y_sTAFyFiycSaW8E,10624
|
|
4
|
+
cyvest/cyvest.py,sha256=suyBM5SPg9Xbo9QVIUP_exaIqTEp19SC1wEkKlpM6w8,50657
|
|
5
|
+
cyvest/investigation.py,sha256=fbhzVe0D4Es4pWiApafwRVTQ-8ZjX10NPNG6I7UC0ys,61653
|
|
6
|
+
cyvest/io_rich.py,sha256=Y0JT3IBE8_agRYZB5-dZSROCVt2nVFN8scu4xFktyO4,42111
|
|
7
|
+
cyvest/io_schema.py,sha256=Z-vlhrbWnGJcIm8k-XVSmzv1vbSvl7I6Z2GV55iU1fc,1296
|
|
8
|
+
cyvest/io_serialization.py,sha256=lr672zxl8-2ZVn10kxpIHVwMuBaF1hbDyR6xreZW1bM,18127
|
|
9
|
+
cyvest/io_visualization.py,sha256=kPT8cAqZJ3liSA2BzwvQXMaejnk496m0Ck11QXJCQLc,11010
|
|
10
|
+
cyvest/keys.py,sha256=xNpSZW5VDzeqc5F0-66-jHM7_v00wt2A70kvbqRSxwA,5678
|
|
11
|
+
cyvest/level_score_rules.py,sha256=MoCCYCLsTDCM-OqLV4Ln_GEH7H_tjNpK4tte89IEQeU,2391
|
|
12
|
+
cyvest/levels.py,sha256=1d3YHCbP2aptSGyIwVzLF4rJjAYdpSzmOnzjs7ZLsdg,4556
|
|
13
|
+
cyvest/model.py,sha256=Em3D5kXS5HCUrehjEEOPWXbX1-UVaEf4ebOs5Z1DBAw,18600
|
|
14
|
+
cyvest/model_enums.py,sha256=t7uFDnGcLXLMZdwgHMxk3HU5KHQgqUvBNZ5i_H2_Jvc,2050
|
|
15
|
+
cyvest/model_schema.py,sha256=Xrd10_dRPcXyiotAF6LJHi7iioGqPBvQlCE87vUVvaM,6037
|
|
16
|
+
cyvest/proxies.py,sha256=BcPV5Cy_-KjA_57bh8HNYM5qXGm0rItogZ8Knis7-rY,20313
|
|
17
|
+
cyvest/score.py,sha256=CiD-dcjEVr2oLHPJ4MlcQWv5338Tq-Zn4Q7lXIEq9fY,19807
|
|
18
|
+
cyvest/shared.py,sha256=mXj0CLwqaVzLupzmLFu1JZO-VtQz_2GCAOnkchqQ4a4,19829
|
|
19
|
+
cyvest/stats.py,sha256=X2e9BHqJrfaOHNUt6jShQqy2Pdw4DMdGheRD9WGPmS0,9035
|
|
20
|
+
cyvest/ulid.py,sha256=NA3k6hlgMZyfzLEMJEIMqt73CY2V07Bupqk7gemshDM,1069
|
|
21
|
+
cyvest-5.1.3.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
|
|
22
|
+
cyvest-5.1.3.dist-info/entry_points.txt,sha256=bwtGzs4Eh9i3WEhW4dhxHaYP8qf8qUN4sDnt4xXywUk,44
|
|
23
|
+
cyvest-5.1.3.dist-info/METADATA,sha256=U0hwMGeDHSF7Gkjgt-tZU7oNeTuNDYtBAoni21uLRdg,23269
|
|
24
|
+
cyvest-5.1.3.dist-info/RECORD,,
|