timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
notification_service.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TimeCodeSecurity Notification Orchestration Service.
|
|
3
|
+
|
|
4
|
+
Orchestrates:
|
|
5
|
+
- Phase 15A EventPublisher & InMemoryEventBus
|
|
6
|
+
- Phase 15C NotificationPolicy, InAppNotificationAdapter & WebhookDeliveryAdapter
|
|
7
|
+
- Phase 15D DurableDeliveryResultSink & delivery audit logs
|
|
8
|
+
|
|
9
|
+
Invariants:
|
|
10
|
+
- notification_service.py is orchestration ONLY.
|
|
11
|
+
- Never defines a new event schema.
|
|
12
|
+
- Never duplicates EventPublisher logic.
|
|
13
|
+
- Never bypasses EventBus.
|
|
14
|
+
- Never modifies input ScanResult.
|
|
15
|
+
- Strictly adheres to the Data Safety Rule: NEVER puts webhook secrets, credentials,
|
|
16
|
+
or user.webhook_url into SecurityDomainEvent metadata.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import logging
|
|
22
|
+
from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
|
|
23
|
+
|
|
24
|
+
from event_bus import DispatchResult, InMemoryEventBus
|
|
25
|
+
from event_publisher import EventPublisher
|
|
26
|
+
from event_taxonomy import SecurityDomainEvent
|
|
27
|
+
from in_app_adapter import InAppNotificationAdapter
|
|
28
|
+
from notification_policy import NotificationIntent, NotificationPolicy, PolicyConfig
|
|
29
|
+
from sqlite_delivery_repository import SqliteDeliveryRepository
|
|
30
|
+
from sqlite_notification_repository import SqliteNotificationRepository
|
|
31
|
+
from webhook_adapter import (
|
|
32
|
+
InMemoryDeliveryResultSink,
|
|
33
|
+
WebhookDeliveryAdapter,
|
|
34
|
+
WebhookDeliveryService,
|
|
35
|
+
WebhookDestinationConfig,
|
|
36
|
+
)
|
|
37
|
+
from durable_delivery_sink import DurableDeliveryResultSink
|
|
38
|
+
|
|
39
|
+
logger = logging.getLogger("tcs.notification_service")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class NotificationService:
|
|
43
|
+
"""
|
|
44
|
+
Central orchestration boundary for security notifications.
|
|
45
|
+
Binds EventBus to NotificationPolicy and delivery adapters.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
bus: Optional[InMemoryEventBus] = None,
|
|
51
|
+
event_publisher: Optional[EventPublisher] = None,
|
|
52
|
+
in_app_adapter: Optional[InAppNotificationAdapter] = None,
|
|
53
|
+
webhook_adapter: Optional[WebhookDeliveryAdapter] = None,
|
|
54
|
+
webhook_service: Optional[WebhookDeliveryService] = None,
|
|
55
|
+
durable_sink: Optional[DurableDeliveryResultSink] = None,
|
|
56
|
+
session_factory: Optional[Callable[[], Any]] = None,
|
|
57
|
+
) -> None:
|
|
58
|
+
self.bus = bus or InMemoryEventBus()
|
|
59
|
+
self.event_publisher = event_publisher or EventPublisher(bus=self.bus)
|
|
60
|
+
|
|
61
|
+
# 1. Setup durable delivery sink
|
|
62
|
+
if durable_sink is not None:
|
|
63
|
+
self.durable_sink = durable_sink
|
|
64
|
+
else:
|
|
65
|
+
self.durable_sink = DurableDeliveryResultSink(
|
|
66
|
+
session_factory=session_factory,
|
|
67
|
+
memory_sink=InMemoryDeliveryResultSink(),
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# 2. Setup webhook service and adapter
|
|
71
|
+
self.webhook_service = webhook_service or WebhookDeliveryService(
|
|
72
|
+
max_workers=4,
|
|
73
|
+
max_queue_size=100,
|
|
74
|
+
result_sink=self.durable_sink,
|
|
75
|
+
)
|
|
76
|
+
self.webhook_adapter = webhook_adapter or WebhookDeliveryAdapter(service=self.webhook_service)
|
|
77
|
+
|
|
78
|
+
# 3. Setup in-app adapter
|
|
79
|
+
if in_app_adapter is not None:
|
|
80
|
+
self.in_app_adapter = in_app_adapter
|
|
81
|
+
else:
|
|
82
|
+
repo = SqliteNotificationRepository(session_factory=session_factory)
|
|
83
|
+
self.in_app_adapter = InAppNotificationAdapter(repository=repo)
|
|
84
|
+
|
|
85
|
+
# 4. Destination registry by organization ID
|
|
86
|
+
self._destinations: Dict[int, WebhookDestinationConfig] = {}
|
|
87
|
+
self._policy_configs: Dict[int, PolicyConfig] = {}
|
|
88
|
+
|
|
89
|
+
# 5. Subscribe orchestration handler to EventBus
|
|
90
|
+
self.bus.subscribe(
|
|
91
|
+
handler=self.handle_domain_event,
|
|
92
|
+
event_type=None,
|
|
93
|
+
name="NotificationServiceOrchestrator",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# Destination Registry Contract & Limitation:
|
|
97
|
+
# "Phase 15D supports runtime/in-memory webhook destination registration.
|
|
98
|
+
# Durable webhook destination configuration management is deferred to a
|
|
99
|
+
# future phase."
|
|
100
|
+
# The registry:
|
|
101
|
+
# - remains organization-scoped (Dict[int, WebhookDestinationConfig])
|
|
102
|
+
# - enforces destination.organization_id == intent.organization_id
|
|
103
|
+
# - never receives webhook URL through SecurityDomainEvent metadata
|
|
104
|
+
# - never exposes webhook secrets
|
|
105
|
+
# - remains the only destination lookup mechanism used by webhook delivery
|
|
106
|
+
|
|
107
|
+
def register_destination(self, destination: WebhookDestinationConfig) -> None:
|
|
108
|
+
"""
|
|
109
|
+
Registers an organization-scoped webhook destination in the runtime in-memory registry.
|
|
110
|
+
Phase 15D limitation: runtime/in-memory only. Durable config is deferred.
|
|
111
|
+
"""
|
|
112
|
+
self._destinations[destination.organization_id] = destination
|
|
113
|
+
|
|
114
|
+
def register_policy_config(self, config: PolicyConfig) -> None:
|
|
115
|
+
"""Registers a tenant notification policy configuration."""
|
|
116
|
+
self._policy_configs[config.organization_id] = config
|
|
117
|
+
|
|
118
|
+
def handle_domain_event(self, event: SecurityDomainEvent) -> None:
|
|
119
|
+
"""
|
|
120
|
+
Synchronous EventBus subscriber callback.
|
|
121
|
+
Evaluates policy and dispatches intents to adapters with error isolation.
|
|
122
|
+
"""
|
|
123
|
+
org_id_raw = event.metadata.get("organization_id") if event.metadata else None
|
|
124
|
+
if org_id_raw is None:
|
|
125
|
+
logger.warning("Event %s discarded: missing mandatory organization_id in metadata", event.event_id)
|
|
126
|
+
return
|
|
127
|
+
try:
|
|
128
|
+
org_id = int(org_id_raw)
|
|
129
|
+
except (ValueError, TypeError):
|
|
130
|
+
logger.error("Event %s discarded: invalid organization_id '%s'", event.event_id, org_id_raw)
|
|
131
|
+
return
|
|
132
|
+
|
|
133
|
+
policy_config = self._policy_configs.get(
|
|
134
|
+
org_id,
|
|
135
|
+
PolicyConfig(
|
|
136
|
+
organization_id=org_id,
|
|
137
|
+
min_severity_in_app="LOW",
|
|
138
|
+
min_severity_webhook="HIGH",
|
|
139
|
+
),
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
intents = NotificationPolicy.evaluate(event, policy_config)
|
|
144
|
+
except Exception as exc:
|
|
145
|
+
logger.error("Notification policy evaluation failed for event %s: %s", event.event_id, exc)
|
|
146
|
+
return
|
|
147
|
+
|
|
148
|
+
for intent in intents:
|
|
149
|
+
try:
|
|
150
|
+
if intent.channel == "IN_APP":
|
|
151
|
+
self.in_app_adapter.handle_intent(intent)
|
|
152
|
+
elif intent.channel == "WEBHOOK":
|
|
153
|
+
destination = self._destinations.get(intent.organization_id)
|
|
154
|
+
if destination:
|
|
155
|
+
if destination.organization_id != intent.organization_id:
|
|
156
|
+
logger.error(
|
|
157
|
+
"Tenant boundary violation: destination org %s != intent org %s",
|
|
158
|
+
destination.organization_id,
|
|
159
|
+
intent.organization_id,
|
|
160
|
+
)
|
|
161
|
+
continue
|
|
162
|
+
self.webhook_adapter.dispatch_intent(intent, destination)
|
|
163
|
+
except Exception as exc:
|
|
164
|
+
logger.error(
|
|
165
|
+
"Error dispatching intent %s on channel %s: %s",
|
|
166
|
+
intent.intent_id,
|
|
167
|
+
intent.channel,
|
|
168
|
+
exc,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
def publish_scan_events(
|
|
172
|
+
self,
|
|
173
|
+
scan_result: Mapping[str, Any],
|
|
174
|
+
repository: str,
|
|
175
|
+
scan_id: str,
|
|
176
|
+
metadata: Optional[Mapping[str, Any]] = None,
|
|
177
|
+
policy_thresholds: Optional[Mapping[str, Any]] = None,
|
|
178
|
+
) -> List[Tuple[SecurityDomainEvent, List[DispatchResult]]]:
|
|
179
|
+
"""
|
|
180
|
+
Delegates scan result event publication to the Phase 15A EventPublisher.
|
|
181
|
+
Ensures strict data minimization: no raw webhook secrets in metadata.
|
|
182
|
+
"""
|
|
183
|
+
clean_metadata = dict(metadata) if metadata else {}
|
|
184
|
+
# Enforce Data Safety Rule: Strip any accidental webhook credentials/URLs from event metadata
|
|
185
|
+
clean_metadata.pop("webhook_url", None)
|
|
186
|
+
clean_metadata.pop("secret", None)
|
|
187
|
+
clean_metadata.pop("api_key", None)
|
|
188
|
+
|
|
189
|
+
return self.event_publisher.publish_scan_events(
|
|
190
|
+
scan_result=scan_result,
|
|
191
|
+
repository=repository,
|
|
192
|
+
scan_id=scan_id,
|
|
193
|
+
metadata=clean_metadata,
|
|
194
|
+
policy_thresholds=policy_thresholds,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
def shutdown(self, max_wait_seconds: float = 3.0) -> None:
|
|
198
|
+
"""Shuts down the webhook delivery service workers monotonically."""
|
|
199
|
+
self.webhook_service.shutdown(max_wait_seconds=max_wait_seconds)
|
osv_client.py
ADDED
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TimeCodeSecurity OSV API Client & Cache Layer.
|
|
3
|
+
Queries Open Source Vulnerabilities (osv.dev) for Python packages via /v1/querybatch.
|
|
4
|
+
Includes local caching, batch chunking, bounded retries, and offline fallback.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
import os
|
|
9
|
+
import json
|
|
10
|
+
import time
|
|
11
|
+
import urllib.request
|
|
12
|
+
import urllib.error
|
|
13
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Dict, List, Optional, Tuple, Any, Callable
|
|
17
|
+
|
|
18
|
+
from manifest_parser import normalize_package_name
|
|
19
|
+
|
|
20
|
+
OSV_API_URL = "https://api.osv.dev/v1/querybatch"
|
|
21
|
+
OSV_VULN_URL = "https://api.osv.dev/v1/vulns"
|
|
22
|
+
DEFAULT_BATCH_CHUNK_SIZE = 500
|
|
23
|
+
DEFAULT_TIMEOUT_SECONDS = 10.0
|
|
24
|
+
DEFAULT_MAX_RETRIES = 3
|
|
25
|
+
DEFAULT_BACKOFF_FACTOR = 0.5
|
|
26
|
+
DEFAULT_HYDRATION_WORKERS = 10
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class OSVAffectedRange:
|
|
31
|
+
"""Represents a range or list of versions affected by an advisory."""
|
|
32
|
+
type: str # e.g. "ECOSYSTEM", "SEMVER", "GIT"
|
|
33
|
+
events: Tuple[Dict[str, str], ...] = () # e.g. ({"introduced": "2.3.0"}, {"fixed": "2.31.0"})
|
|
34
|
+
versions: Tuple[str, ...] = () # list of explicitly enumerated version strings
|
|
35
|
+
database_specific: Dict[str, Any] = field(default_factory=dict)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class OSVVulnerability:
|
|
40
|
+
"""Normalized structured representation of an OSV advisory."""
|
|
41
|
+
vuln_id: str # Primary advisory ID (GHSA-..., PYSEC-..., CVE-...)
|
|
42
|
+
package_name: str # Normalized package name
|
|
43
|
+
ecosystem: str = "PyPI"
|
|
44
|
+
summary: str = ""
|
|
45
|
+
details: str = ""
|
|
46
|
+
aliases: Tuple[str, ...] = () # Associated IDs (CVE-..., GHSA-...)
|
|
47
|
+
affected_ranges: Tuple[OSVAffectedRange, ...] = () # Affected version ranges and explicit versions
|
|
48
|
+
severity: Tuple[Dict[str, Any], ...] = () # CVSS vectors/scores
|
|
49
|
+
references: Tuple[Dict[str, str], ...] = () # External advisory links
|
|
50
|
+
database_specific: Dict[str, Any] = field(default_factory=dict)
|
|
51
|
+
raw: Dict[str, Any] = field(default_factory=dict) # Raw original OSV record
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class OSVQueryResult:
|
|
56
|
+
"""Result of querying advisories for a specific package."""
|
|
57
|
+
package_name: str # Normalized package name
|
|
58
|
+
vulnerabilities: List[OSVVulnerability] = field(default_factory=list)
|
|
59
|
+
source: str = "network" # "network", "cache", "offline_fallback", "error"
|
|
60
|
+
error: Optional[str] = None
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def has_vulnerabilities(self) -> bool:
|
|
64
|
+
return len(self.vulnerabilities) > 0
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _parse_osv_vuln(package_name: str, raw_vuln: Dict[str, Any]) -> OSVVulnerability:
|
|
68
|
+
"""Translates a raw OSV JSON vulnerability record into an OSVVulnerability object."""
|
|
69
|
+
vuln_id = raw_vuln.get("id", "UNKNOWN_ID")
|
|
70
|
+
summary = raw_vuln.get("summary", "")
|
|
71
|
+
details = raw_vuln.get("details", "")
|
|
72
|
+
aliases = tuple(raw_vuln.get("aliases", []))
|
|
73
|
+
severity = tuple(raw_vuln.get("severity", []))
|
|
74
|
+
references = tuple(raw_vuln.get("references", []))
|
|
75
|
+
db_specific = raw_vuln.get("database_specific", {})
|
|
76
|
+
|
|
77
|
+
ranges_list: List[OSVAffectedRange] = []
|
|
78
|
+
for affected in raw_vuln.get("affected", []):
|
|
79
|
+
aff_pkg = affected.get("package", {})
|
|
80
|
+
aff_pkg_name = normalize_package_name(aff_pkg.get("name", ""))
|
|
81
|
+
aff_eco = aff_pkg.get("ecosystem", "PyPI")
|
|
82
|
+
|
|
83
|
+
# Include if ecosystem is PyPI and package name matches or is unspecified
|
|
84
|
+
if aff_eco.lower() == "pypi" and (not aff_pkg_name or aff_pkg_name == package_name):
|
|
85
|
+
explicit_versions = tuple(affected.get("versions", []))
|
|
86
|
+
for r in affected.get("ranges", []):
|
|
87
|
+
ranges_list.append(OSVAffectedRange(
|
|
88
|
+
type=r.get("type", "ECOSYSTEM"),
|
|
89
|
+
events=tuple(r.get("events", [])),
|
|
90
|
+
versions=explicit_versions,
|
|
91
|
+
database_specific=r.get("database_specific", {})
|
|
92
|
+
))
|
|
93
|
+
|
|
94
|
+
# If no range blocks were provided but explicit versions exist
|
|
95
|
+
if not affected.get("ranges") and explicit_versions:
|
|
96
|
+
ranges_list.append(OSVAffectedRange(
|
|
97
|
+
type="VERSIONS",
|
|
98
|
+
events=(),
|
|
99
|
+
versions=explicit_versions,
|
|
100
|
+
database_specific=affected.get("database_specific", {})
|
|
101
|
+
))
|
|
102
|
+
|
|
103
|
+
return OSVVulnerability(
|
|
104
|
+
vuln_id=vuln_id,
|
|
105
|
+
package_name=package_name,
|
|
106
|
+
ecosystem="PyPI",
|
|
107
|
+
summary=summary,
|
|
108
|
+
details=details,
|
|
109
|
+
aliases=aliases,
|
|
110
|
+
affected_ranges=tuple(ranges_list),
|
|
111
|
+
severity=severity,
|
|
112
|
+
references=references,
|
|
113
|
+
database_specific=db_specific,
|
|
114
|
+
raw=raw_vuln
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _is_stub(vuln_dict: Dict[str, Any]) -> bool:
|
|
119
|
+
"""Returns True if the vulnerability record is an unhydrated stub (missing affected ranges)."""
|
|
120
|
+
return "affected" not in vuln_dict and "schema_version" not in vuln_dict
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class OSVClient:
|
|
124
|
+
"""
|
|
125
|
+
Offline-capable, batch-chunking OSV API Client for Python/PyPI packages.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
def __init__(
|
|
129
|
+
self,
|
|
130
|
+
api_url: str = OSV_API_URL,
|
|
131
|
+
vuln_api_url: str = OSV_VULN_URL,
|
|
132
|
+
cache_file: Optional[str] = None,
|
|
133
|
+
timeout: float = DEFAULT_TIMEOUT_SECONDS,
|
|
134
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
135
|
+
backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
|
|
136
|
+
batch_chunk_size: int = DEFAULT_BATCH_CHUNK_SIZE,
|
|
137
|
+
offline_mode: bool = False,
|
|
138
|
+
offline_fixtures: Optional[Dict[str, List[Dict[str, Any]]]] = None,
|
|
139
|
+
http_requester: Optional[Callable[[str, bytes, float], bytes]] = None,
|
|
140
|
+
http_get_requester: Optional[Callable[[str, float], bytes]] = None,
|
|
141
|
+
max_workers: int = DEFAULT_HYDRATION_WORKERS
|
|
142
|
+
):
|
|
143
|
+
self.api_url = api_url
|
|
144
|
+
self.vuln_api_url = vuln_api_url
|
|
145
|
+
self.cache_file = str(Path(os.path.expanduser(str(cache_file))).resolve()) if cache_file else None
|
|
146
|
+
self.timeout = timeout
|
|
147
|
+
self.max_retries = max_retries
|
|
148
|
+
self.backoff_factor = backoff_factor
|
|
149
|
+
self.batch_chunk_size = batch_chunk_size
|
|
150
|
+
self.offline_mode = offline_mode
|
|
151
|
+
self.max_workers = max_workers
|
|
152
|
+
self._memory_cache: Dict[str, List[Dict[str, Any]]] = {}
|
|
153
|
+
self._vuln_cache: Dict[str, Dict[str, Any]] = {}
|
|
154
|
+
self._http_requester = http_requester or self._default_http_post
|
|
155
|
+
self._http_get_requester = http_get_requester or self._default_http_get
|
|
156
|
+
|
|
157
|
+
# Load offline fixtures if provided
|
|
158
|
+
if offline_fixtures:
|
|
159
|
+
for k, v in offline_fixtures.items():
|
|
160
|
+
self._memory_cache[normalize_package_name(k)] = v
|
|
161
|
+
|
|
162
|
+
# Load local disk cache if configured
|
|
163
|
+
if self.cache_file:
|
|
164
|
+
self._load_disk_cache()
|
|
165
|
+
|
|
166
|
+
def _load_disk_cache(self) -> None:
|
|
167
|
+
"""Loads cached responses from the local cache file."""
|
|
168
|
+
if not self.cache_file:
|
|
169
|
+
return
|
|
170
|
+
p = Path(os.path.expanduser(str(self.cache_file))).resolve()
|
|
171
|
+
if p.is_file():
|
|
172
|
+
try:
|
|
173
|
+
data = json.loads(p.read_text(encoding="utf-8"))
|
|
174
|
+
if isinstance(data, dict):
|
|
175
|
+
for k, v in data.items():
|
|
176
|
+
if isinstance(v, list):
|
|
177
|
+
self._memory_cache[normalize_package_name(k)] = v
|
|
178
|
+
except Exception:
|
|
179
|
+
pass # Tolerate corrupt cache file without crashing
|
|
180
|
+
|
|
181
|
+
def _save_disk_cache(self) -> None:
|
|
182
|
+
"""Persists memory cache to the local cache file."""
|
|
183
|
+
if not self.cache_file:
|
|
184
|
+
return
|
|
185
|
+
try:
|
|
186
|
+
p = Path(os.path.expanduser(str(self.cache_file))).resolve()
|
|
187
|
+
# Hard invariant: ensure we never write to a literal '~' directory in the working directory
|
|
188
|
+
if "~" in p.parts:
|
|
189
|
+
return
|
|
190
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
191
|
+
p.write_text(json.dumps(self._memory_cache, indent=2), encoding="utf-8")
|
|
192
|
+
except Exception:
|
|
193
|
+
pass # Non-fatal if disk write fails
|
|
194
|
+
|
|
195
|
+
def _default_http_post(self, url: str, payload_bytes: bytes, timeout: float) -> bytes:
|
|
196
|
+
"""Standard library HTTP POST request handler."""
|
|
197
|
+
req = urllib.request.Request(
|
|
198
|
+
url,
|
|
199
|
+
data=payload_bytes,
|
|
200
|
+
headers={
|
|
201
|
+
"Content-Type": "application/json",
|
|
202
|
+
"User-Agent": "TimeCodeSecurity-SCA/1.0"
|
|
203
|
+
},
|
|
204
|
+
method="POST"
|
|
205
|
+
)
|
|
206
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
207
|
+
return response.read()
|
|
208
|
+
|
|
209
|
+
def _default_http_get(self, url: str, timeout: float) -> bytes:
|
|
210
|
+
"""Standard library HTTP GET request handler."""
|
|
211
|
+
req = urllib.request.Request(
|
|
212
|
+
url,
|
|
213
|
+
headers={
|
|
214
|
+
"Accept": "application/json",
|
|
215
|
+
"User-Agent": "TimeCodeSecurity-SCA/1.0"
|
|
216
|
+
},
|
|
217
|
+
method="GET"
|
|
218
|
+
)
|
|
219
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
220
|
+
return response.read()
|
|
221
|
+
|
|
222
|
+
def _fetch_vuln_detail(self, vuln_id: str) -> Optional[Dict[str, Any]]:
|
|
223
|
+
"""
|
|
224
|
+
Fetches full advisory details for a specific vulnerability ID from /v1/vulns/{id}.
|
|
225
|
+
Checks _vuln_cache first. Retries on 5xx/network errors with exponential backoff.
|
|
226
|
+
Fails fast on 4xx. Returns None if unable to fetch.
|
|
227
|
+
"""
|
|
228
|
+
if not vuln_id:
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
if vuln_id in self._vuln_cache:
|
|
232
|
+
return self._vuln_cache[vuln_id]
|
|
233
|
+
|
|
234
|
+
if self.offline_mode:
|
|
235
|
+
return None
|
|
236
|
+
|
|
237
|
+
url = f"{self.vuln_api_url.rstrip('/')}/{vuln_id}"
|
|
238
|
+
|
|
239
|
+
for attempt in range(self.max_retries):
|
|
240
|
+
try:
|
|
241
|
+
raw_bytes = self._http_get_requester(url, self.timeout)
|
|
242
|
+
data = json.loads(raw_bytes.decode("utf-8"))
|
|
243
|
+
if isinstance(data, dict) and data.get("id"):
|
|
244
|
+
self._vuln_cache[vuln_id] = data
|
|
245
|
+
return data
|
|
246
|
+
return None
|
|
247
|
+
except urllib.error.HTTPError as http_err:
|
|
248
|
+
status = http_err.code
|
|
249
|
+
if 400 <= status < 500:
|
|
250
|
+
return None
|
|
251
|
+
if attempt < self.max_retries - 1:
|
|
252
|
+
sleep_time = self.backoff_factor * (2 ** attempt)
|
|
253
|
+
if sleep_time > 0:
|
|
254
|
+
time.sleep(sleep_time)
|
|
255
|
+
else:
|
|
256
|
+
return None
|
|
257
|
+
except Exception:
|
|
258
|
+
if attempt < self.max_retries - 1:
|
|
259
|
+
sleep_time = self.backoff_factor * (2 ** attempt)
|
|
260
|
+
if sleep_time > 0:
|
|
261
|
+
time.sleep(sleep_time)
|
|
262
|
+
else:
|
|
263
|
+
return None
|
|
264
|
+
|
|
265
|
+
return None
|
|
266
|
+
|
|
267
|
+
def _hydrate_vulns(self, vulns: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
268
|
+
"""
|
|
269
|
+
Hydrates any unhydrated vulnerability stubs in the list.
|
|
270
|
+
Uses ThreadPoolExecutor for concurrent hydration if multiple stubs exist.
|
|
271
|
+
"""
|
|
272
|
+
stubs = [v for v in vulns if _is_stub(v) and v.get("id")]
|
|
273
|
+
if not stubs or self.offline_mode:
|
|
274
|
+
return vulns
|
|
275
|
+
|
|
276
|
+
hydrated_map: Dict[str, Dict[str, Any]] = {}
|
|
277
|
+
unique_ids = list({v["id"] for v in stubs if v.get("id")})
|
|
278
|
+
|
|
279
|
+
if len(unique_ids) == 1 or self.max_workers <= 1:
|
|
280
|
+
for vid in unique_ids:
|
|
281
|
+
detail = self._fetch_vuln_detail(vid)
|
|
282
|
+
if detail:
|
|
283
|
+
hydrated_map[vid] = detail
|
|
284
|
+
else:
|
|
285
|
+
with ThreadPoolExecutor(max_workers=min(self.max_workers, len(unique_ids))) as executor:
|
|
286
|
+
future_to_id = {executor.submit(self._fetch_vuln_detail, vid): vid for vid in unique_ids}
|
|
287
|
+
for future in as_completed(future_to_id):
|
|
288
|
+
vid = future_to_id[future]
|
|
289
|
+
try:
|
|
290
|
+
detail = future.result()
|
|
291
|
+
if detail:
|
|
292
|
+
hydrated_map[vid] = detail
|
|
293
|
+
except Exception:
|
|
294
|
+
pass
|
|
295
|
+
|
|
296
|
+
result: List[Dict[str, Any]] = []
|
|
297
|
+
for v in vulns:
|
|
298
|
+
vid = v.get("id")
|
|
299
|
+
if vid and vid in hydrated_map:
|
|
300
|
+
result.append(hydrated_map[vid])
|
|
301
|
+
else:
|
|
302
|
+
result.append(v)
|
|
303
|
+
return result
|
|
304
|
+
|
|
305
|
+
def query_package(self, package_name: str) -> OSVQueryResult:
|
|
306
|
+
"""Convenience method to query a single package."""
|
|
307
|
+
results = self.query_packages([package_name])
|
|
308
|
+
norm = normalize_package_name(package_name)
|
|
309
|
+
return results.get(norm, OSVQueryResult(package_name=norm, source="error", error="Not found"))
|
|
310
|
+
|
|
311
|
+
def query_packages(self, package_names: List[str]) -> Dict[str, OSVQueryResult]:
|
|
312
|
+
"""
|
|
313
|
+
Queries OSV database for multiple packages using batch requests.
|
|
314
|
+
Handles caching, chunking, bounded retries, and offline fallback.
|
|
315
|
+
"""
|
|
316
|
+
results_map: Dict[str, OSVQueryResult] = {}
|
|
317
|
+
pending_names: List[str] = []
|
|
318
|
+
cache_dirty = False
|
|
319
|
+
|
|
320
|
+
# 1. Check local cache and offline fixtures
|
|
321
|
+
for raw_name in package_names:
|
|
322
|
+
norm_name = normalize_package_name(raw_name)
|
|
323
|
+
if not norm_name:
|
|
324
|
+
continue
|
|
325
|
+
|
|
326
|
+
if norm_name in self._memory_cache:
|
|
327
|
+
raw_vulns = self._memory_cache[norm_name]
|
|
328
|
+
if not self.offline_mode and any(_is_stub(v) for v in raw_vulns if v.get("id")):
|
|
329
|
+
raw_vulns = self._hydrate_vulns(raw_vulns)
|
|
330
|
+
self._memory_cache[norm_name] = raw_vulns
|
|
331
|
+
cache_dirty = True
|
|
332
|
+
parsed = [_parse_osv_vuln(norm_name, v) for v in raw_vulns]
|
|
333
|
+
results_map[norm_name] = OSVQueryResult(
|
|
334
|
+
package_name=norm_name,
|
|
335
|
+
vulnerabilities=parsed,
|
|
336
|
+
source="cache"
|
|
337
|
+
)
|
|
338
|
+
else:
|
|
339
|
+
if norm_name not in pending_names:
|
|
340
|
+
pending_names.append(norm_name)
|
|
341
|
+
|
|
342
|
+
# 2. If in offline mode, do not attempt network requests
|
|
343
|
+
if self.offline_mode:
|
|
344
|
+
for name in pending_names:
|
|
345
|
+
results_map[name] = OSVQueryResult(
|
|
346
|
+
package_name=name,
|
|
347
|
+
vulnerabilities=[],
|
|
348
|
+
source="offline_fallback",
|
|
349
|
+
error="Offline mode active; package not found in local cache"
|
|
350
|
+
)
|
|
351
|
+
if cache_dirty:
|
|
352
|
+
self._save_disk_cache()
|
|
353
|
+
return results_map
|
|
354
|
+
|
|
355
|
+
if not pending_names:
|
|
356
|
+
if cache_dirty:
|
|
357
|
+
self._save_disk_cache()
|
|
358
|
+
return results_map
|
|
359
|
+
|
|
360
|
+
# 3. Chunk pending packages into batches (up to batch_chunk_size)
|
|
361
|
+
for i in range(0, len(pending_names), self.batch_chunk_size):
|
|
362
|
+
chunk = pending_names[i:i + self.batch_chunk_size]
|
|
363
|
+
chunk_results = self._fetch_batch_chunk(chunk)
|
|
364
|
+
|
|
365
|
+
for name, (raw_vulns, source, err_msg) in chunk_results.items():
|
|
366
|
+
if source == "network":
|
|
367
|
+
raw_vulns = self._hydrate_vulns(raw_vulns)
|
|
368
|
+
self._memory_cache[name] = raw_vulns
|
|
369
|
+
cache_dirty = True
|
|
370
|
+
parsed_vulns = [_parse_osv_vuln(name, v) for v in raw_vulns]
|
|
371
|
+
results_map[name] = OSVQueryResult(
|
|
372
|
+
package_name=name,
|
|
373
|
+
vulnerabilities=parsed_vulns,
|
|
374
|
+
source=source,
|
|
375
|
+
error=err_msg
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
if cache_dirty:
|
|
379
|
+
self._save_disk_cache()
|
|
380
|
+
|
|
381
|
+
return results_map
|
|
382
|
+
|
|
383
|
+
def _fetch_batch_chunk(self, chunk: List[str]) -> Dict[str, Tuple[List[Dict[str, Any]], str, Optional[str]]]:
|
|
384
|
+
"""
|
|
385
|
+
Executes an HTTP POST to /v1/querybatch for a single chunk of packages.
|
|
386
|
+
Applies exponential backoff retries for 5xx errors/timeouts and fails fast on 4xx.
|
|
387
|
+
"""
|
|
388
|
+
chunk_map: Dict[str, Tuple[List[Dict[str, Any]], str, Optional[str]]] = {}
|
|
389
|
+
queries = [{"package": {"name": name, "ecosystem": "PyPI"}} for name in chunk]
|
|
390
|
+
payload = {"queries": queries}
|
|
391
|
+
payload_bytes = json.dumps(payload).encode("utf-8")
|
|
392
|
+
|
|
393
|
+
for attempt in range(self.max_retries):
|
|
394
|
+
try:
|
|
395
|
+
raw_response = self._http_requester(self.api_url, payload_bytes, self.timeout)
|
|
396
|
+
data = json.loads(raw_response.decode("utf-8"))
|
|
397
|
+
batch_results = data.get("results", [])
|
|
398
|
+
|
|
399
|
+
for idx, name in enumerate(chunk):
|
|
400
|
+
vulns_list: List[Dict[str, Any]] = []
|
|
401
|
+
if idx < len(batch_results):
|
|
402
|
+
entry = batch_results[idx]
|
|
403
|
+
vulns_list = entry.get("vulns", []) if isinstance(entry, dict) else []
|
|
404
|
+
chunk_map[name] = (vulns_list, "network", None)
|
|
405
|
+
|
|
406
|
+
return chunk_map
|
|
407
|
+
|
|
408
|
+
except urllib.error.HTTPError as http_err:
|
|
409
|
+
status = http_err.code
|
|
410
|
+
# Fail fast on 4xx client errors
|
|
411
|
+
if 400 <= status < 500:
|
|
412
|
+
for name in chunk:
|
|
413
|
+
chunk_map[name] = ([], "error", f"HTTP {status}: Client error")
|
|
414
|
+
return chunk_map
|
|
415
|
+
|
|
416
|
+
# 5xx server errors: retry with backoff
|
|
417
|
+
if attempt < self.max_retries - 1:
|
|
418
|
+
sleep_time = self.backoff_factor * (2 ** attempt)
|
|
419
|
+
if sleep_time > 0:
|
|
420
|
+
time.sleep(sleep_time)
|
|
421
|
+
else:
|
|
422
|
+
for name in chunk:
|
|
423
|
+
chunk_map[name] = ([], "error", f"HTTP {status}: Server error after {self.max_retries} attempts")
|
|
424
|
+
return chunk_map
|
|
425
|
+
|
|
426
|
+
except (urllib.error.URLError, TimeoutError, Exception) as net_err:
|
|
427
|
+
# Network / timeout errors: retry with backoff
|
|
428
|
+
if attempt < self.max_retries - 1:
|
|
429
|
+
sleep_time = self.backoff_factor * (2 ** attempt)
|
|
430
|
+
if sleep_time > 0:
|
|
431
|
+
time.sleep(sleep_time)
|
|
432
|
+
else:
|
|
433
|
+
for name in chunk:
|
|
434
|
+
chunk_map[name] = ([], "offline_fallback", f"Network error: {net_err}")
|
|
435
|
+
return chunk_map
|
|
436
|
+
|
|
437
|
+
return chunk_map
|
remediation/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TimeCodeSecurity - Vector D Remediation Engine Package.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from remediation.contracts import (
|
|
6
|
+
PatchStatus,
|
|
7
|
+
RemediationRule,
|
|
8
|
+
RemediationRecord,
|
|
9
|
+
FIXTURE_SEMANTIC_CONTRACTS,
|
|
10
|
+
)
|
|
11
|
+
from remediation.base import BaseRemediationTransformer
|
|
12
|
+
from remediation.cwe89_sqli import Cwe89ParameterizeTransformer
|
|
13
|
+
from remediation.cwe78_cmdi import Cwe78CmdInjectionTransformer
|
|
14
|
+
from remediation.cwe22_path_traversal import Cwe22PathTraversalTransformer
|
|
15
|
+
from remediation.dispatcher import RuleDispatcher
|
|
16
|
+
from remediation.verification import RemediationVerifier
|
|
17
|
+
from remediation.patch_engine import RemediationEngine
|
|
18
|
+
from remediation.orchestrator import (
|
|
19
|
+
ProjectRemediationResult,
|
|
20
|
+
format_remediation_section,
|
|
21
|
+
remediate_project,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"PatchStatus",
|
|
26
|
+
"RemediationRule",
|
|
27
|
+
"RemediationRecord",
|
|
28
|
+
"FIXTURE_SEMANTIC_CONTRACTS",
|
|
29
|
+
"BaseRemediationTransformer",
|
|
30
|
+
"Cwe89ParameterizeTransformer",
|
|
31
|
+
"Cwe78CmdInjectionTransformer",
|
|
32
|
+
"Cwe22PathTraversalTransformer",
|
|
33
|
+
"RuleDispatcher",
|
|
34
|
+
"RemediationVerifier",
|
|
35
|
+
"RemediationEngine",
|
|
36
|
+
"ProjectRemediationResult",
|
|
37
|
+
"format_remediation_section",
|
|
38
|
+
"remediate_project",
|
|
39
|
+
]
|