cyvest 4.4.0__py3-none-any.whl → 5.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cyvest might be problematic. Click here for more details.
- cyvest/__init__.py +24 -5
- cyvest/cli.py +63 -1
- cyvest/compare.py +310 -0
- cyvest/cyvest.py +253 -181
- cyvest/investigation.py +276 -243
- cyvest/io_rich.py +141 -54
- cyvest/io_schema.py +1 -1
- cyvest/io_serialization.py +90 -91
- cyvest/keys.py +61 -18
- cyvest/model.py +55 -43
- cyvest/model_schema.py +9 -9
- cyvest/proxies.py +48 -50
- cyvest/shared.py +19 -19
- cyvest/stats.py +11 -36
- {cyvest-4.4.0.dist-info → cyvest-5.1.0.dist-info}/METADATA +105 -12
- cyvest-5.1.0.dist-info/RECORD +24 -0
- {cyvest-4.4.0.dist-info → cyvest-5.1.0.dist-info}/WHEEL +1 -1
- cyvest-4.4.0.dist-info/RECORD +0 -23
- {cyvest-4.4.0.dist-info → cyvest-5.1.0.dist-info}/entry_points.txt +0 -0
cyvest/io_rich.py
CHANGED
|
@@ -50,8 +50,8 @@ def _sort_key_by_score(item: Any) -> tuple[Decimal, str]:
|
|
|
50
50
|
except (TypeError, ValueError, InvalidOperation):
|
|
51
51
|
decimal_score = Decimal(0)
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
return (-decimal_score,
|
|
53
|
+
item_name = getattr(item, "check_name", "")
|
|
54
|
+
return (-decimal_score, item_name)
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
def _get_direction_symbol(rel: Relationship, reversed_edge: bool) -> str:
|
|
@@ -373,59 +373,14 @@ def display_summary(
|
|
|
373
373
|
table.add_column("Score", justify="right")
|
|
374
374
|
table.add_column("Level", justify="center")
|
|
375
375
|
|
|
376
|
-
# Checks section
|
|
377
|
-
rule = Rule("[bold magenta]CHECKS[/bold magenta]")
|
|
378
|
-
table.add_row(rule, "-", "-")
|
|
379
|
-
|
|
380
|
-
# Organize checks by scope
|
|
381
|
-
checks_by_scope: dict[str, list[Any]] = {}
|
|
382
|
-
for check in cv.check_get_all().values():
|
|
383
|
-
if check.level in resolved_excluded_levels:
|
|
384
|
-
continue
|
|
385
|
-
if check.scope not in checks_by_scope:
|
|
386
|
-
checks_by_scope[check.scope] = []
|
|
387
|
-
checks_by_scope[check.scope].append(check)
|
|
388
|
-
|
|
389
|
-
for scope_name, checks in checks_by_scope.items():
|
|
390
|
-
scope_rule = Align(f"[bold magenta]{scope_name}[/bold magenta]", align="left")
|
|
391
|
-
table.add_row(scope_rule, "-", "-")
|
|
392
|
-
checks = sorted(checks, key=_sort_key_by_score)
|
|
393
|
-
for check in checks:
|
|
394
|
-
color_level = get_color_level(check.level)
|
|
395
|
-
color_score = get_color_score(check.score)
|
|
396
|
-
name = f" {check.check_id}"
|
|
397
|
-
score = f"[{color_score}]{check.score_display}[/{color_score}]"
|
|
398
|
-
level = f"[{color_level}]{check.level.name}[/{color_level}]"
|
|
399
|
-
table.add_row(name, score, level)
|
|
400
|
-
|
|
401
|
-
# Containers section (if any)
|
|
402
|
-
if cv.container_get_all():
|
|
403
|
-
table.add_section()
|
|
404
|
-
rule = Rule("[bold magenta]CONTAINERS[/bold magenta]")
|
|
405
|
-
table.add_row(rule, "-", "-")
|
|
406
|
-
|
|
407
|
-
for container in cv.container_get_all().values():
|
|
408
|
-
agg_score = container.get_aggregated_score()
|
|
409
|
-
agg_level = container.get_aggregated_level()
|
|
410
|
-
color_level = get_color_level(agg_level)
|
|
411
|
-
color_score = get_color_score(agg_score)
|
|
412
|
-
|
|
413
|
-
name = f" {container.path}"
|
|
414
|
-
score = f"[{color_score}]{agg_score:.2f}[/{color_score}]"
|
|
415
|
-
level = f"[{color_level}]{agg_level.name}[/{color_level}]"
|
|
416
|
-
table.add_row(name, score, level)
|
|
417
|
-
|
|
418
376
|
# Checks by level section
|
|
419
|
-
|
|
420
|
-
rule = Rule("[bold magenta]BY LEVEL[/bold magenta]")
|
|
377
|
+
rule = Rule(f"[bold magenta]CHECKS[/bold magenta]: {len(cv.check_get_all())} checks")
|
|
421
378
|
table.add_row(rule, "-", "-")
|
|
422
379
|
|
|
423
|
-
for level_enum in
|
|
380
|
+
for level_enum in sorted(Level, reverse=True):
|
|
424
381
|
if level_enum in resolved_excluded_levels:
|
|
425
382
|
continue
|
|
426
|
-
checks = [
|
|
427
|
-
c for c in cv.check_get_all().values() if c.level == level_enum and c.level not in resolved_excluded_levels
|
|
428
|
-
]
|
|
383
|
+
checks = [c for c in cv.check_get_all().values() if c.level == level_enum]
|
|
429
384
|
checks = sorted(checks, key=_sort_key_by_score)
|
|
430
385
|
if checks:
|
|
431
386
|
color_level = get_color_level(level_enum)
|
|
@@ -437,11 +392,28 @@ def display_summary(
|
|
|
437
392
|
|
|
438
393
|
for check in checks:
|
|
439
394
|
color_score = get_color_score(check.score)
|
|
440
|
-
name = f" {check.
|
|
395
|
+
name = f" {check.check_name}"
|
|
441
396
|
score = f"[{color_score}]{check.score_display}[/{color_score}]"
|
|
442
397
|
level = f"[{color_level}]{check.level.name}[/{color_level}]"
|
|
443
398
|
table.add_row(name, score, level)
|
|
444
399
|
|
|
400
|
+
# Tags section (if any)
|
|
401
|
+
if cv.tag_get_all():
|
|
402
|
+
table.add_section()
|
|
403
|
+
rule = Rule(f"[bold magenta]TAGS[/bold magenta]: {len(cv.tag_get_all())} tags")
|
|
404
|
+
table.add_row(rule, "-", "-")
|
|
405
|
+
|
|
406
|
+
for tag in cv.tag_get_all().values():
|
|
407
|
+
agg_score = tag.get_aggregated_score()
|
|
408
|
+
agg_level = tag.get_aggregated_level()
|
|
409
|
+
color_level = get_color_level(agg_level)
|
|
410
|
+
color_score = get_color_score(agg_score)
|
|
411
|
+
|
|
412
|
+
name = f" {tag.name}"
|
|
413
|
+
score = f"[{color_score}]{agg_score:.2f}[/{color_score}]"
|
|
414
|
+
level = f"[{color_level}]{agg_level.name}[/{color_level}]"
|
|
415
|
+
table.add_row(name, score, level)
|
|
416
|
+
|
|
445
417
|
# Enrichments section (if any)
|
|
446
418
|
if cv.enrichment_get_all():
|
|
447
419
|
table.add_section()
|
|
@@ -558,11 +530,11 @@ def display_statistics(cv: Cyvest, rich_print: Callable[[Any], None]) -> None:
|
|
|
558
530
|
# Check statistics table
|
|
559
531
|
rich_print("")
|
|
560
532
|
check_table = Table(title="Check Statistics")
|
|
561
|
-
check_table.add_column("
|
|
533
|
+
check_table.add_column("Level", style="cyan")
|
|
562
534
|
check_table.add_column("Count", justify="right")
|
|
563
535
|
|
|
564
|
-
for
|
|
565
|
-
check_table.add_row(
|
|
536
|
+
for level, count in stats.checks_by_level.items():
|
|
537
|
+
check_table.add_row(level, str(count))
|
|
566
538
|
|
|
567
539
|
rich_print(check_table)
|
|
568
540
|
|
|
@@ -577,3 +549,118 @@ def display_statistics(cv: Cyvest, rich_print: Callable[[Any], None]) -> None:
|
|
|
577
549
|
ti_table.add_row(source, str(count))
|
|
578
550
|
|
|
579
551
|
rich_print(ti_table)
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _format_level_score(
|
|
555
|
+
level: Level | None,
|
|
556
|
+
score: Decimal | None,
|
|
557
|
+
score_rule: str | None = None,
|
|
558
|
+
) -> str:
|
|
559
|
+
"""Format level and score for display."""
|
|
560
|
+
if level is None and score is None and not score_rule:
|
|
561
|
+
return "[dim]-[/dim]"
|
|
562
|
+
|
|
563
|
+
parts: list[str] = []
|
|
564
|
+
if level:
|
|
565
|
+
color = get_color_level(level)
|
|
566
|
+
parts.append(f"[{color}]{level.name}[/{color}]")
|
|
567
|
+
|
|
568
|
+
if score_rule:
|
|
569
|
+
parts.append(score_rule)
|
|
570
|
+
elif score is not None:
|
|
571
|
+
color = get_color_score(score)
|
|
572
|
+
parts.append(f"[{color}]{_format_score_decimal(score)}[/{color}]")
|
|
573
|
+
|
|
574
|
+
return " ".join(parts) if parts else "[dim]-[/dim]"
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def display_diff(
|
|
578
|
+
diffs: list,
|
|
579
|
+
rich_print: Callable[[Any], None],
|
|
580
|
+
title: str = "Diff",
|
|
581
|
+
) -> None:
|
|
582
|
+
"""
|
|
583
|
+
Display investigation diff in a rich table with tree structure.
|
|
584
|
+
|
|
585
|
+
Args:
|
|
586
|
+
diffs: List of DiffItem objects representing differences
|
|
587
|
+
rich_print: A rich renderable handler called with renderables for output
|
|
588
|
+
title: Title for the diff table
|
|
589
|
+
"""
|
|
590
|
+
# Import here to avoid circular dependency
|
|
591
|
+
from cyvest.compare import DiffStatus
|
|
592
|
+
|
|
593
|
+
# Count diffs by status
|
|
594
|
+
added = sum(1 for d in diffs if d.status == DiffStatus.ADDED)
|
|
595
|
+
removed = sum(1 for d in diffs if d.status == DiffStatus.REMOVED)
|
|
596
|
+
mismatch = sum(1 for d in diffs if d.status == DiffStatus.MISMATCH)
|
|
597
|
+
|
|
598
|
+
# Build caption with combined legend and counts
|
|
599
|
+
caption = (
|
|
600
|
+
f"[green]+ {added}[/green] added | [red]- {removed}[/red] removed | [yellow]\u2717 {mismatch}[/yellow] mismatch"
|
|
601
|
+
)
|
|
602
|
+
|
|
603
|
+
table = Table(title=title, caption=caption)
|
|
604
|
+
table.add_column("Key")
|
|
605
|
+
table.add_column("Expected", justify="center")
|
|
606
|
+
table.add_column("Actual", justify="center")
|
|
607
|
+
table.add_column("Status", justify="center", width=8)
|
|
608
|
+
|
|
609
|
+
status_styles = {
|
|
610
|
+
DiffStatus.ADDED: "green",
|
|
611
|
+
DiffStatus.REMOVED: "red",
|
|
612
|
+
DiffStatus.MISMATCH: "yellow",
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
for idx, diff in enumerate(diffs):
|
|
616
|
+
# Add section separator between checks (except before first)
|
|
617
|
+
if idx > 0:
|
|
618
|
+
table.add_section()
|
|
619
|
+
|
|
620
|
+
status_style = status_styles.get(diff.status, "white")
|
|
621
|
+
|
|
622
|
+
# Check row (main row)
|
|
623
|
+
expected_str = _format_level_score(diff.expected_level, diff.expected_score, diff.expected_score_rule)
|
|
624
|
+
actual_str = _format_level_score(diff.actual_level, diff.actual_score)
|
|
625
|
+
|
|
626
|
+
table.add_row(
|
|
627
|
+
escape(diff.key),
|
|
628
|
+
expected_str,
|
|
629
|
+
actual_str,
|
|
630
|
+
f"[{status_style}]{diff.status.value}[/{status_style}]",
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
# Observable rows (indented with └──)
|
|
634
|
+
for obs_idx, obs in enumerate(diff.observable_diffs):
|
|
635
|
+
is_last_obs = obs_idx == len(diff.observable_diffs) - 1
|
|
636
|
+
obs_prefix = "└──" if is_last_obs else "├──"
|
|
637
|
+
|
|
638
|
+
obs_label = obs.observable_key
|
|
639
|
+
obs_expected = _format_level_score(obs.expected_level, obs.expected_score)
|
|
640
|
+
obs_actual = _format_level_score(obs.actual_level, obs.actual_score)
|
|
641
|
+
|
|
642
|
+
table.add_row(
|
|
643
|
+
f"{obs_prefix} [cyan]{escape(obs_label)}[/cyan]",
|
|
644
|
+
obs_expected,
|
|
645
|
+
obs_actual,
|
|
646
|
+
"",
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
# Threat intel rows (indented further with │ └── or └──)
|
|
650
|
+
for ti_idx, ti in enumerate(obs.threat_intel_diffs):
|
|
651
|
+
is_last_ti = ti_idx == len(obs.threat_intel_diffs) - 1
|
|
652
|
+
# Use │ continuation if not last observable, else spaces
|
|
653
|
+
continuation = "│ " if not is_last_obs else " "
|
|
654
|
+
ti_prefix = "└──" if is_last_ti else "├──"
|
|
655
|
+
|
|
656
|
+
ti_expected = _format_level_score(ti.expected_level, ti.expected_score)
|
|
657
|
+
ti_actual = _format_level_score(ti.actual_level, ti.actual_score)
|
|
658
|
+
|
|
659
|
+
table.add_row(
|
|
660
|
+
f"{continuation}{ti_prefix} [magenta]{escape(ti.source)}[/magenta]",
|
|
661
|
+
ti_expected,
|
|
662
|
+
ti_actual,
|
|
663
|
+
"",
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
rich_print(table)
|
cyvest/io_schema.py
CHANGED
|
@@ -26,7 +26,7 @@ def get_investigation_schema() -> dict[str, Any]:
|
|
|
26
26
|
matches the actual `model_dump()` output structure.
|
|
27
27
|
|
|
28
28
|
The returned schema automatically includes all referenced entity types
|
|
29
|
-
(Observable, Check, ThreatIntel, Enrichment,
|
|
29
|
+
(Observable, Check, ThreatIntel, Enrichment, Tag, InvestigationWhitelist)
|
|
30
30
|
in the `$defs` section.
|
|
31
31
|
|
|
32
32
|
Returns:
|
cyvest/io_serialization.py
CHANGED
|
@@ -12,7 +12,7 @@ from pathlib import Path
|
|
|
12
12
|
from typing import TYPE_CHECKING, Any
|
|
13
13
|
|
|
14
14
|
from cyvest.levels import Level, normalize_level
|
|
15
|
-
from cyvest.model import AuditEvent, Check,
|
|
15
|
+
from cyvest.model import AuditEvent, Check, Enrichment, Observable, Relationship, Tag, ThreatIntel
|
|
16
16
|
from cyvest.model_enums import ObservableType
|
|
17
17
|
from cyvest.model_schema import InvestigationSchema
|
|
18
18
|
from cyvest.score import ScoreMode
|
|
@@ -37,18 +37,14 @@ def serialize_investigation(inv: Investigation, *, include_audit_log: bool = Tru
|
|
|
37
37
|
Returns:
|
|
38
38
|
InvestigationSchema instance (use .model_dump() for dict)
|
|
39
39
|
"""
|
|
40
|
-
inv.
|
|
40
|
+
inv._rebuild_all_check_links()
|
|
41
41
|
observables = dict(inv.get_all_observables())
|
|
42
42
|
threat_intels = dict(inv.get_all_threat_intels())
|
|
43
43
|
enrichments = dict(inv.get_all_enrichments())
|
|
44
|
-
|
|
44
|
+
tags = dict(inv.get_all_tags())
|
|
45
45
|
|
|
46
|
-
#
|
|
47
|
-
|
|
48
|
-
for check in inv.get_all_checks().values():
|
|
49
|
-
if check.scope not in checks_by_scope:
|
|
50
|
-
checks_by_scope[check.scope] = []
|
|
51
|
-
checks_by_scope[check.scope].append(check)
|
|
46
|
+
# Get all checks
|
|
47
|
+
checks = dict(inv.get_all_checks())
|
|
52
48
|
|
|
53
49
|
# Get root type
|
|
54
50
|
root = inv.get_root()
|
|
@@ -64,10 +60,10 @@ def serialize_investigation(inv: Investigation, *, include_audit_log: bool = Tru
|
|
|
64
60
|
whitelists=list(inv.get_whitelists()),
|
|
65
61
|
audit_log=inv.get_audit_log() if include_audit_log else None,
|
|
66
62
|
observables=observables,
|
|
67
|
-
checks=
|
|
63
|
+
checks=checks,
|
|
68
64
|
threat_intels=threat_intels,
|
|
69
65
|
enrichments=enrichments,
|
|
70
|
-
|
|
66
|
+
tags=tags,
|
|
71
67
|
stats=inv.get_statistics(),
|
|
72
68
|
data_extraction={
|
|
73
69
|
"root_type": root_type_value,
|
|
@@ -95,7 +91,7 @@ def save_investigation_json(inv: Investigation, filepath: str | Path, *, include
|
|
|
95
91
|
|
|
96
92
|
def generate_markdown_report(
|
|
97
93
|
inv: Investigation,
|
|
98
|
-
|
|
94
|
+
include_tags: bool = False,
|
|
99
95
|
include_enrichments: bool = False,
|
|
100
96
|
include_observables: bool = True,
|
|
101
97
|
) -> str:
|
|
@@ -104,7 +100,7 @@ def generate_markdown_report(
|
|
|
104
100
|
|
|
105
101
|
Args:
|
|
106
102
|
inv: Investigation
|
|
107
|
-
|
|
103
|
+
include_tags: Include tags section in the report (default: False)
|
|
108
104
|
include_enrichments: Include enrichments section in the report (default: False)
|
|
109
105
|
include_observables: Include observables section in the report (default: True)
|
|
110
106
|
|
|
@@ -150,19 +146,16 @@ def generate_markdown_report(
|
|
|
150
146
|
lines.append(f" - Justification: {entry.justification}")
|
|
151
147
|
lines.append("")
|
|
152
148
|
|
|
153
|
-
# Checks
|
|
154
|
-
lines.append("## Checks
|
|
149
|
+
# Checks
|
|
150
|
+
lines.append("## Checks")
|
|
151
|
+
lines.append("")
|
|
152
|
+
for check in inv.get_all_checks().values():
|
|
153
|
+
if check.level != Level.NONE:
|
|
154
|
+
lines.append(f"- **{check.check_name}**: Score: {check.score_display}, Level: {check.level.name}")
|
|
155
|
+
lines.append(f" - Description: {check.description}")
|
|
156
|
+
if check.comment:
|
|
157
|
+
lines.append(f" - Comment: {check.comment}")
|
|
155
158
|
lines.append("")
|
|
156
|
-
for scope, _count in inv.get_statistics().checks_by_scope.items():
|
|
157
|
-
lines.append(f"### {scope}")
|
|
158
|
-
lines.append("")
|
|
159
|
-
for check in inv.get_all_checks().values():
|
|
160
|
-
if check.scope == scope and check.level != Level.NONE:
|
|
161
|
-
lines.append(f"- **{check.check_id}**: Score: {check.score_display}, Level: {check.level.name}")
|
|
162
|
-
lines.append(f" - Description: {check.description}")
|
|
163
|
-
if check.comment:
|
|
164
|
-
lines.append(f" - Comment: {check.comment}")
|
|
165
|
-
lines.append("")
|
|
166
159
|
|
|
167
160
|
# Observables
|
|
168
161
|
if include_observables and inv.get_all_observables():
|
|
@@ -205,17 +198,17 @@ def generate_markdown_report(
|
|
|
205
198
|
lines.append(f"- **Data:** {json.dumps(enr.data, indent=2)}")
|
|
206
199
|
lines.append("")
|
|
207
200
|
|
|
208
|
-
#
|
|
209
|
-
if
|
|
210
|
-
lines.append("##
|
|
201
|
+
# Tags
|
|
202
|
+
if include_tags and inv.get_all_tags():
|
|
203
|
+
lines.append("## Tags")
|
|
211
204
|
lines.append("")
|
|
212
|
-
for
|
|
213
|
-
lines.append(f"### {
|
|
214
|
-
lines.append(f"- **Description:** {
|
|
215
|
-
lines.append(f"- **
|
|
216
|
-
lines.append(f"- **Aggregated
|
|
217
|
-
lines.append(f"- **
|
|
218
|
-
lines.append(f"- **
|
|
205
|
+
for tag in inv.get_all_tags().values():
|
|
206
|
+
lines.append(f"### {tag.name}")
|
|
207
|
+
lines.append(f"- **Description:** {tag.description}")
|
|
208
|
+
lines.append(f"- **Direct Score:** {tag.get_direct_score():.2f}")
|
|
209
|
+
lines.append(f"- **Aggregated Score:** {inv.get_tag_aggregated_score(tag.name):.2f}")
|
|
210
|
+
lines.append(f"- **Aggregated Level:** {inv.get_tag_aggregated_level(tag.name).name}")
|
|
211
|
+
lines.append(f"- **Direct Checks:** {len(tag.checks)}")
|
|
219
212
|
lines.append("")
|
|
220
213
|
|
|
221
214
|
return "\n".join(lines)
|
|
@@ -224,7 +217,7 @@ def generate_markdown_report(
|
|
|
224
217
|
def save_investigation_markdown(
|
|
225
218
|
inv: Investigation,
|
|
226
219
|
filepath: str | Path,
|
|
227
|
-
|
|
220
|
+
include_tags: bool = False,
|
|
228
221
|
include_enrichments: bool = False,
|
|
229
222
|
include_observables: bool = True,
|
|
230
223
|
) -> None:
|
|
@@ -234,21 +227,21 @@ def save_investigation_markdown(
|
|
|
234
227
|
Args:
|
|
235
228
|
inv: Investigation to save
|
|
236
229
|
filepath: Path to save the Markdown file
|
|
237
|
-
|
|
230
|
+
include_tags: Include tags section in the report (default: False)
|
|
238
231
|
include_enrichments: Include enrichments section in the report (default: False)
|
|
239
232
|
include_observables: Include observables section in the report (default: True)
|
|
240
233
|
"""
|
|
241
|
-
markdown = generate_markdown_report(inv,
|
|
234
|
+
markdown = generate_markdown_report(inv, include_tags, include_enrichments, include_observables)
|
|
242
235
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
243
236
|
f.write(markdown)
|
|
244
237
|
|
|
245
238
|
|
|
246
|
-
def
|
|
239
|
+
def load_investigation_dict(data: dict[str, Any]) -> Cyvest:
|
|
247
240
|
"""
|
|
248
|
-
Load an investigation from a JSON
|
|
241
|
+
Load an investigation from a dictionary (parsed JSON) into a Cyvest object.
|
|
249
242
|
|
|
250
243
|
Args:
|
|
251
|
-
|
|
244
|
+
data: Dictionary containing the serialized investigation data
|
|
252
245
|
|
|
253
246
|
Returns:
|
|
254
247
|
Reconstructed Cyvest investigation
|
|
@@ -256,9 +249,6 @@ def load_investigation_json(filepath: str | Path) -> Cyvest:
|
|
|
256
249
|
from cyvest.cyvest import Cyvest
|
|
257
250
|
from cyvest.investigation import Investigation
|
|
258
251
|
|
|
259
|
-
with open(filepath, encoding="utf-8") as handle:
|
|
260
|
-
data = json.load(handle)
|
|
261
|
-
|
|
262
252
|
investigation_id = data.get("investigation_id")
|
|
263
253
|
if not isinstance(investigation_id, str) or not investigation_id.strip():
|
|
264
254
|
raise ValueError("Serialized investigation must include 'investigation_id'.")
|
|
@@ -380,35 +370,32 @@ def load_investigation_json(filepath: str | Path) -> Cyvest:
|
|
|
380
370
|
cv._investigation.add_threat_intel(ti, observable)
|
|
381
371
|
|
|
382
372
|
# Checks - leverage Pydantic model_validate
|
|
383
|
-
for
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
}
|
|
410
|
-
check = Check.model_validate(check_data)
|
|
411
|
-
cv._investigation.add_check(check)
|
|
373
|
+
for check_info in data.get("checks", {}).values():
|
|
374
|
+
raw_links = check_info.get("observable_links", []) or []
|
|
375
|
+
normalized_links = []
|
|
376
|
+
for link in raw_links:
|
|
377
|
+
if isinstance(link, dict):
|
|
378
|
+
normalized_links.append(
|
|
379
|
+
{
|
|
380
|
+
"observable_key": link.get("observable_key", ""),
|
|
381
|
+
"propagation_mode": link.get("propagation_mode", "LOCAL_ONLY"),
|
|
382
|
+
}
|
|
383
|
+
)
|
|
384
|
+
else:
|
|
385
|
+
normalized_links.append(link)
|
|
386
|
+
check_data = {
|
|
387
|
+
"check_name": check_info.get("check_name", ""),
|
|
388
|
+
"description": check_info.get("description", ""),
|
|
389
|
+
"comment": check_info.get("comment", ""),
|
|
390
|
+
"extra": check_info.get("extra", {}),
|
|
391
|
+
"score": Decimal(str(check_info.get("score", 0))),
|
|
392
|
+
"level": check_info.get("level", "NONE"),
|
|
393
|
+
"origin_investigation_id": check_info.get("origin_investigation_id") or cv._investigation.investigation_id,
|
|
394
|
+
"observable_links": normalized_links,
|
|
395
|
+
"key": check_info.get("key", ""),
|
|
396
|
+
}
|
|
397
|
+
check = Check.model_validate(check_data)
|
|
398
|
+
cv._investigation.add_check(check)
|
|
412
399
|
|
|
413
400
|
# Enrichments - leverage Pydantic model_validate
|
|
414
401
|
for enr_info in data.get("enrichments", {}).values():
|
|
@@ -421,31 +408,27 @@ def load_investigation_json(filepath: str | Path) -> Cyvest:
|
|
|
421
408
|
enrichment = Enrichment.model_validate(enr_data)
|
|
422
409
|
cv._investigation.add_enrichment(enrichment)
|
|
423
410
|
|
|
424
|
-
#
|
|
425
|
-
def
|
|
426
|
-
|
|
427
|
-
"
|
|
428
|
-
"description":
|
|
429
|
-
"key":
|
|
411
|
+
# Tags
|
|
412
|
+
def build_tag(tag_info: dict[str, Any]) -> Tag:
|
|
413
|
+
tag_data = {
|
|
414
|
+
"name": tag_info.get("name", ""),
|
|
415
|
+
"description": tag_info.get("description", ""),
|
|
416
|
+
"key": tag_info.get("key", ""),
|
|
430
417
|
}
|
|
431
|
-
|
|
432
|
-
|
|
418
|
+
tag = Tag.model_validate(tag_data)
|
|
419
|
+
tag = cv._investigation.add_tag(tag)
|
|
433
420
|
|
|
434
|
-
for check_key in
|
|
421
|
+
for check_key in tag_info.get("checks", []):
|
|
435
422
|
check = cv._investigation.get_check(check_key)
|
|
436
423
|
if check:
|
|
437
|
-
cv._investigation.
|
|
438
|
-
|
|
439
|
-
for sub_info in container_info.get("sub_containers", {}).values():
|
|
440
|
-
sub_container = build_container(sub_info)
|
|
441
|
-
cv._investigation.add_sub_container(container.key, sub_container.key)
|
|
424
|
+
cv._investigation.add_check_to_tag(tag.key, check.key)
|
|
442
425
|
|
|
443
|
-
return
|
|
426
|
+
return tag
|
|
444
427
|
|
|
445
|
-
for
|
|
446
|
-
|
|
428
|
+
for tag_info in data.get("tags", {}).values():
|
|
429
|
+
build_tag(tag_info)
|
|
447
430
|
|
|
448
|
-
cv._investigation.
|
|
431
|
+
cv._investigation._rebuild_all_check_links()
|
|
449
432
|
|
|
450
433
|
audit_log = []
|
|
451
434
|
for event_info in data.get("audit_log", []) or []:
|
|
@@ -457,3 +440,19 @@ def load_investigation_json(filepath: str | Path) -> Cyvest:
|
|
|
457
440
|
cv._investigation._audit_enabled = True
|
|
458
441
|
|
|
459
442
|
return cv
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def load_investigation_json(filepath: str | Path) -> Cyvest:
|
|
446
|
+
"""
|
|
447
|
+
Load an investigation from a JSON file into a Cyvest object.
|
|
448
|
+
|
|
449
|
+
Args:
|
|
450
|
+
filepath: Path to the JSON file
|
|
451
|
+
|
|
452
|
+
Returns:
|
|
453
|
+
Reconstructed Cyvest investigation
|
|
454
|
+
"""
|
|
455
|
+
with open(filepath, encoding="utf-8") as handle:
|
|
456
|
+
data = json.load(handle)
|
|
457
|
+
|
|
458
|
+
return load_investigation_dict(data)
|
cyvest/keys.py
CHANGED
|
@@ -56,22 +56,20 @@ def generate_observable_key(obs_type: str, value: str) -> str:
|
|
|
56
56
|
return f"obs:{normalized_type}:{normalized_value}"
|
|
57
57
|
|
|
58
58
|
|
|
59
|
-
def generate_check_key(
|
|
59
|
+
def generate_check_key(check_name: str) -> str:
|
|
60
60
|
"""
|
|
61
61
|
Generate a unique key for a check.
|
|
62
62
|
|
|
63
|
-
Format: chk:{
|
|
63
|
+
Format: chk:{check_name}
|
|
64
64
|
|
|
65
65
|
Args:
|
|
66
|
-
|
|
67
|
-
scope: Scope of the check
|
|
66
|
+
check_name: Name of the check
|
|
68
67
|
|
|
69
68
|
Returns:
|
|
70
69
|
Unique check key
|
|
71
70
|
"""
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
return f"chk:{normalized_id}:{normalized_scope}"
|
|
71
|
+
normalized_name = _normalize_value(check_name)
|
|
72
|
+
return f"chk:{normalized_name}"
|
|
75
73
|
|
|
76
74
|
|
|
77
75
|
def generate_threat_intel_key(source: str, observable_key: str) -> str:
|
|
@@ -111,22 +109,67 @@ def generate_enrichment_key(name: str, context: str = "") -> str:
|
|
|
111
109
|
return f"enr:{normalized_name}"
|
|
112
110
|
|
|
113
111
|
|
|
114
|
-
def
|
|
112
|
+
def generate_tag_key(name: str) -> str:
|
|
115
113
|
"""
|
|
116
|
-
Generate a unique key for a
|
|
114
|
+
Generate a unique key for a tag.
|
|
117
115
|
|
|
118
|
-
Format:
|
|
116
|
+
Format: tag:{normalized_name}
|
|
119
117
|
|
|
120
118
|
Args:
|
|
121
|
-
|
|
119
|
+
name: Name of the tag (uses : as hierarchy delimiter)
|
|
122
120
|
|
|
123
121
|
Returns:
|
|
124
|
-
Unique
|
|
122
|
+
Unique tag key
|
|
125
123
|
"""
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
124
|
+
normalized_name = _normalize_value(name)
|
|
125
|
+
return f"tag:{normalized_name}"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def get_tag_ancestors(name: str) -> list[str]:
|
|
129
|
+
"""
|
|
130
|
+
Get all ancestor tag names from a hierarchical tag name.
|
|
131
|
+
|
|
132
|
+
Uses ":" as the hierarchy delimiter.
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
name: Tag name (e.g., "header:auth:dkim")
|
|
136
|
+
|
|
137
|
+
Returns:
|
|
138
|
+
List of ancestor names (e.g., ["header", "header:auth"])
|
|
139
|
+
"""
|
|
140
|
+
parts = name.split(":")
|
|
141
|
+
return [":".join(parts[: i + 1]) for i in range(len(parts) - 1)]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def is_tag_child_of(child_name: str, parent_name: str) -> bool:
|
|
145
|
+
"""
|
|
146
|
+
Check if a tag is a direct child of another tag.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
child_name: Potential child tag name
|
|
150
|
+
parent_name: Potential parent tag name
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
True if child_name is a direct child of parent_name
|
|
154
|
+
"""
|
|
155
|
+
if not child_name.startswith(parent_name + ":"):
|
|
156
|
+
return False
|
|
157
|
+
remaining = child_name[len(parent_name) + 1 :]
|
|
158
|
+
return ":" not in remaining
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def is_tag_descendant_of(descendant_name: str, ancestor_name: str) -> bool:
|
|
162
|
+
"""
|
|
163
|
+
Check if a tag is a descendant (child, grandchild, etc.) of another tag.
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
descendant_name: Potential descendant tag name
|
|
167
|
+
ancestor_name: Potential ancestor tag name
|
|
168
|
+
|
|
169
|
+
Returns:
|
|
170
|
+
True if descendant_name is a descendant of ancestor_name
|
|
171
|
+
"""
|
|
172
|
+
return descendant_name.startswith(ancestor_name + ":")
|
|
130
173
|
|
|
131
174
|
|
|
132
175
|
def parse_key_type(key: str) -> str | None:
|
|
@@ -137,7 +180,7 @@ def parse_key_type(key: str) -> str | None:
|
|
|
137
180
|
key: The key to parse
|
|
138
181
|
|
|
139
182
|
Returns:
|
|
140
|
-
Type prefix (obs, chk, ti, enr,
|
|
183
|
+
Type prefix (obs, chk, ti, enr, tag) or None if invalid
|
|
141
184
|
"""
|
|
142
185
|
if ":" in key:
|
|
143
186
|
return key.split(":", 1)[0]
|
|
@@ -185,7 +228,7 @@ def validate_key(key: str, expected_type: str | None = None) -> bool:
|
|
|
185
228
|
return False
|
|
186
229
|
|
|
187
230
|
key_type = parse_key_type(key)
|
|
188
|
-
if key_type not in ("obs", "chk", "ti", "enr", "
|
|
231
|
+
if key_type not in ("obs", "chk", "ti", "enr", "tag"):
|
|
189
232
|
return False
|
|
190
233
|
|
|
191
234
|
if expected_type and key_type != expected_type:
|