detecti-cli 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. detecti/__init__.py +0 -0
  2. detecti/cli.py +649 -0
  3. detecti/config.py +188 -0
  4. detecti/core/__init__.py +1 -0
  5. detecti/core/database/__init__.py +5 -0
  6. detecti/core/database/config_db.py +73 -0
  7. detecti/core/database/schema.py +136 -0
  8. detecti/core/database/storage.py +1388 -0
  9. detecti/core/engine.py +1032 -0
  10. detecti/core/models.py +278 -0
  11. detecti/data/config.sqlite +0 -0
  12. detecti/data/dbs/.gitkeep +2 -0
  13. detecti/data/dbs/example.com.sqlite +0 -0
  14. detecti/modules/__init__.py +29 -0
  15. detecti/modules/base.py +57 -0
  16. detecti/modules/censys.py +813 -0
  17. detecti/modules/crtsh.py +98 -0
  18. detecti/modules/exploitdb.py +138 -0
  19. detecti/modules/masscan.py +561 -0
  20. detecti/modules/nuclei.py +449 -0
  21. detecti/modules/nvd.py +300 -0
  22. detecti/modules/reverse_whois.py +225 -0
  23. detecti/modules/shodan.py +412 -0
  24. detecti/reporters/__init__.py +7 -0
  25. detecti/reporters/csv_reporter.py +74 -0
  26. detecti/reporters/html_reporter.py +356 -0
  27. detecti/reporters/json_reporter.py +26 -0
  28. detecti/reporters/markdown_reporter.py +203 -0
  29. detecti/utils/__init__.py +1 -0
  30. detecti/utils/http.py +294 -0
  31. detecti/utils/logger.py +378 -0
  32. detecti/utils/setup.py +453 -0
  33. detecti/web/__init__.py +6 -0
  34. detecti/web/api/__init__.py +1 -0
  35. detecti/web/api/auth.py +109 -0
  36. detecti/web/api/graph_builder.py +901 -0
  37. detecti/web/api/routes.py +1602 -0
  38. detecti/web/process_manager.py +283 -0
  39. detecti/web/server.py +183 -0
  40. detecti/web/static/android-chrome-192x192.png +0 -0
  41. detecti/web/static/android-chrome-512x512.png +0 -0
  42. detecti/web/static/apple-touch-icon.png +0 -0
  43. detecti/web/static/css/__init__.py +1 -0
  44. detecti/web/static/css/dashboard.css +3802 -0
  45. detecti/web/static/favicon-16x16.png +0 -0
  46. detecti/web/static/favicon-32x32.png +0 -0
  47. detecti/web/static/favicon.ico +0 -0
  48. detecti/web/static/img/DetecTI_Security_Logo.png +0 -0
  49. detecti/web/static/img/detecti-ico.png +0 -0
  50. detecti/web/static/index.html +677 -0
  51. detecti/web/static/js/__init__.py +1 -0
  52. detecti/web/static/js/api.js +177 -0
  53. detecti/web/static/js/cytoscape-cose-bilkent.js +458 -0
  54. detecti/web/static/js/cytoscape-dagre.js +397 -0
  55. detecti/web/static/js/cytoscape.min.js +31 -0
  56. detecti/web/static/js/dagre.min.js +3809 -0
  57. detecti/web/static/js/graph.js +7439 -0
  58. detecti/web/static/js/lucide.min.js +12 -0
  59. detecti/web/static/login.html +290 -0
  60. detecti/web/static/site.webmanifest +1 -0
  61. detecti_cli-2.0.0.dist-info/METADATA +554 -0
  62. detecti_cli-2.0.0.dist-info/RECORD +64 -0
  63. detecti_cli-2.0.0.dist-info/WHEEL +4 -0
  64. detecti_cli-2.0.0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,813 @@
1
+ """Censys Platform API (v3) Asset and Host Intelligence Module."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import ipaddress
7
+ import logging
8
+ import os
9
+ import time
10
+ import uuid
11
+ from typing import Any, Dict, List, Optional
12
+ import requests
13
+
14
+ from detecti.config import settings
15
+ from detecti.core.models import (
16
+ Finding,
17
+ FindingType,
18
+ HostInfoData,
19
+ PortData,
20
+ VulnerabilityData,
21
+ )
22
+ from detecti.modules.base import BaseModule
23
+ from detecti.utils.http import AsyncHTTPClient, http_client
24
+
25
+ logger = logging.getLogger("detecti.censys")
26
+
27
+
28
+ def is_valid_uuid(val: Optional[str]) -> bool:
29
+ """Check if a string represents a valid UUID (required by Censys for Organization ID)."""
30
+ if not val or not isinstance(val, str):
31
+ return False
32
+ try:
33
+ uuid.UUID(val.strip())
34
+ return True
35
+ except (ValueError, AttributeError):
36
+ return False
37
+
38
+
39
+ class CensysAPIError(Exception):
40
+ """Base exception for Censys Platform API errors."""
41
+ pass
42
+
43
+
44
+ class CensysAuthError(CensysAPIError):
45
+ """Exception for Authentication/Authorization errors (401/403)."""
46
+ pass
47
+
48
+
49
+ class CensysRateLimitError(CensysAPIError):
50
+ """Exception for Rate Limit exceeded (429)."""
51
+ pass
52
+
53
+
54
+ class CensysQuotaExhaustedError(CensysAPIError):
55
+ """Exception for API quota/balance exhausted (422 with insufficient balance)."""
56
+ pass
57
+
58
+
59
+ class CensysPlatformClient:
60
+ """Synchronous reference client for integration with Censys Platform API v3."""
61
+
62
+ BASE_URL = "https://api.platform.censys.io/v3"
63
+
64
+ def __init__(self, pat_token: Optional[str] = None, org_id: Optional[str] = None):
65
+ self.pat_token = pat_token or settings.censys_pat_token or os.getenv("CENSYS_PAT_TOKEN")
66
+ self.org_id = org_id or settings.censys_org_id or os.getenv("CENSYS_ORG_ID")
67
+
68
+ if not self.pat_token:
69
+ raise ValueError("O token PAT (CENSYS_PAT_TOKEN) é obrigatório para autenticação.")
70
+
71
+ self.session = requests.Session()
72
+ self.session.headers.update({
73
+ "Authorization": f"Bearer {self.pat_token}",
74
+ "Accept": "application/json",
75
+ "Content-Type": "application/json",
76
+ })
77
+ if self.org_id and is_valid_uuid(self.org_id):
78
+ self.session.headers.update({"X-Organization-ID": self.org_id.strip()})
79
+
80
+ def _request(
81
+ self,
82
+ method: str,
83
+ endpoint: str,
84
+ params: Optional[Dict[str, Any]] = None,
85
+ json_data: Optional[Dict[str, Any]] = None,
86
+ max_retries: int = 3,
87
+ ) -> Dict[str, Any]:
88
+ """Execute HTTP request with exponential backoff and error handling."""
89
+ url = f"{self.BASE_URL}{endpoint}"
90
+
91
+ for attempt in range(max_retries):
92
+ try:
93
+ response = self.session.request(method, url, params=params, json=json_data)
94
+
95
+ if response.status_code == 200:
96
+ return response.json()
97
+ elif response.status_code in (401, 403):
98
+ raise CensysAuthError(f"Erro de Autenticação/Permissão [{response.status_code}]: {response.text}")
99
+ elif response.status_code == 429:
100
+ if attempt < max_retries - 1:
101
+ time.sleep(2 ** attempt)
102
+ continue
103
+ raise CensysRateLimitError("Limite de taxa excedido (429).")
104
+ elif response.status_code == 422:
105
+ # Check if it's a quota exhaustion error first
106
+ try:
107
+ error_data = response.json()
108
+ if isinstance(error_data, dict) and "errors" in error_data:
109
+ for error in error_data.get("errors", []):
110
+ if isinstance(error, dict) and "insufficient balance" in error.get("message", "").lower():
111
+ raise CensysQuotaExhaustedError("API quota/balance exhausted")
112
+ except Exception:
113
+ pass # If we can't parse the error, treat it as a regular validation error
114
+
115
+ raise CensysAPIError(f"Erro de validação ou query CenQL (422): {response.text}")
116
+ else:
117
+ response.raise_for_status()
118
+ except requests.RequestException as exc:
119
+ if attempt >= max_retries - 1:
120
+ raise CensysAPIError(f"Falha na requisição Censys após múltiplas tentativas: {exc}")
121
+ time.sleep(2 ** attempt)
122
+
123
+ raise CensysAPIError("Falha na requisição após múltiplas tentativas.")
124
+
125
+ def get_host(self, ip: str) -> Dict[str, Any]:
126
+ """Obtém detalhes de um host específico por IP via Censys Platform API v3."""
127
+ return self._request("GET", f"/global/asset/host/{ip}")
128
+
129
+ def search_query(self, query: str, page_size: int = 100, cursor: Optional[str] = None) -> Dict[str, Any]:
130
+ """Executa uma busca unificada usando a linguagem CenQL."""
131
+ payload: Dict[str, Any] = {"query": query, "page_size": page_size}
132
+ if cursor:
133
+ payload["cursor"] = cursor
134
+ return self._request("POST", "/global/search/query", json_data=payload)
135
+
136
+ def aggregate_search(self, query: str, field: str, num_buckets: int = 10) -> Dict[str, Any]:
137
+ """Calcula estatísticas agregadas por campos específicos."""
138
+ payload = {"query": query, "field": field, "num_buckets": num_buckets}
139
+ return self._request("POST", "/global/search/aggregate", json_data=payload)
140
+
141
+ def get_certificate(self, fingerprint: str) -> Dict[str, Any]:
142
+ """Retorna informações detalhadas de um certificado pelo fingerprint SHA-256."""
143
+ return self._request("GET", f"/global/asset/certificate/{fingerprint}")
144
+
145
+ def convert_legacy_query(self, legacy_query: str) -> Dict[str, Any]:
146
+ """Converte consultas da API Legada (v1/v2) para a sintaxe CenQL."""
147
+ payload = {"query": legacy_query}
148
+ return self._request("POST", "/global/search/convert", json_data=payload)
149
+
150
+
151
+ class CensysModule(BaseModule):
152
+ """Censys Platform API (v3) asynchronous collector supporting CenQL search, host asset lookups, and CIDR ranges."""
153
+
154
+ name: str = "censys"
155
+ description: str = "Censys Platform API v3 internet-wide asset & host scanner"
156
+ category: str = "recon"
157
+
158
+ def __init__(
159
+ self,
160
+ client: Optional[AsyncHTTPClient] = None,
161
+ pat_token: Optional[str] = None,
162
+ org_id: Optional[str] = None,
163
+ base_url: Optional[str] = None,
164
+ progress_callback: Optional[Any] = None,
165
+ **kwargs: Any,
166
+ ):
167
+ super().__init__(client=client, progress_callback=progress_callback)
168
+ from config import is_placeholder_key
169
+ raw_pat = pat_token or settings.censys_pat_token or os.getenv("CENSYS_PAT_TOKEN")
170
+ self.pat_token = None if is_placeholder_key(raw_pat) else raw_pat
171
+ raw_org = org_id or settings.censys_org_id or os.getenv("CENSYS_ORG_ID")
172
+ self.org_id = None if is_placeholder_key(raw_org) else raw_org
173
+ self.base_url = (base_url or settings.censys_platform_api_url or "https://api.platform.censys.io/v3").rstrip("/")
174
+ self._quota_exhausted = False # Flag to skip further API calls when quota is exhausted
175
+ self._auth_failed = False # Flag to skip further API calls when credentials are invalid
176
+
177
+ def is_configured(self) -> bool:
178
+ """Check if valid Censys API credentials (PAT token or legacy ID/Secret) are set."""
179
+ if self._auth_failed:
180
+ return False
181
+ from config import is_placeholder_key
182
+ has_pat = bool(self.pat_token and not is_placeholder_key(self.pat_token))
183
+ has_legacy = bool(
184
+ settings.censys_api_id
185
+ and not is_placeholder_key(settings.censys_api_id)
186
+ and settings.censys_api_secret
187
+ and not is_placeholder_key(settings.censys_api_secret)
188
+ )
189
+ return has_pat or has_legacy
190
+
191
+ async def validate_credentials(self) -> bool:
192
+ """Perform a non-intrusive pre-flight authentication verification check."""
193
+ is_valid, _ = await self.validate_credentials_detailed()
194
+ return is_valid
195
+
196
+ async def validate_credentials_detailed(self) -> tuple[bool, str]:
197
+ """Perform a fast pre-flight authentication check returning validity and status."""
198
+ if not self.is_configured():
199
+ return False, "Not Configured"
200
+ url = f"{self.base_url}/global/asset/host/8.8.8.8"
201
+ headers = self._get_auth_headers(accept_header="application/vnd.censys.api.v3.host.v1+json")
202
+ try:
203
+ resp = await self.http_client.get(
204
+ url=url,
205
+ headers=headers,
206
+ timeout=4.0,
207
+ max_retries=1,
208
+ max_retry_delay=2.0,
209
+ raise_for_status=False,
210
+ )
211
+ if resp.status_code == 200:
212
+ return True, "Active & Valid"
213
+ elif resp.status_code in (401, 403):
214
+ self._auth_failed = True
215
+ logger.debug(f"Censys credential validation failed (HTTP {resp.status_code}).")
216
+ return False, f"Invalid / Unauthorized (HTTP {resp.status_code})"
217
+ elif resp.status_code == 429:
218
+ return False, "Rate Limited / Throttled (HTTP 429)"
219
+ else:
220
+ return False, f"API Error (HTTP {resp.status_code})"
221
+ except Exception as exc:
222
+ logger.debug(f"Censys credential pre-check encountered network exception: {exc}")
223
+ return False, f"Network / Timeout Error ({type(exc).__name__})"
224
+
225
+ def _get_auth_headers(self, accept_header: str = "application/json") -> Dict[str, str]:
226
+ """Generate Platform API v3 Bearer token (or legacy fallback) and Organization headers."""
227
+ headers: Dict[str, str] = {
228
+ "Accept": accept_header,
229
+ "Content-Type": "application/json",
230
+ }
231
+
232
+ pat = self.pat_token or settings.censys_pat_token or os.getenv("CENSYS_PAT_TOKEN")
233
+ if pat:
234
+ headers["Authorization"] = f"Bearer {pat}"
235
+ elif settings.censys_api_id and settings.censys_api_secret:
236
+ auth_bytes = f"{settings.censys_api_id}:{settings.censys_api_secret}".encode("utf-8")
237
+ b64_auth = base64.b64encode(auth_bytes).decode("utf-8")
238
+ headers["Authorization"] = f"Basic {b64_auth}"
239
+
240
+ org_id = self.org_id or settings.censys_org_id or os.getenv("CENSYS_ORG_ID")
241
+ if org_id and is_valid_uuid(org_id):
242
+ headers["X-Organization-ID"] = org_id.strip()
243
+ elif org_id:
244
+ logger.debug(
245
+ f"Ignoring CENSYS_ORG_ID='{org_id}' because it is not a valid UUID. "
246
+ f"Free accounts operate without Organization ID."
247
+ )
248
+
249
+ return headers
250
+
251
+ async def run(
252
+ self,
253
+ target: str,
254
+ context: Optional[Dict[str, Any]] = None,
255
+ ) -> List[Finding]:
256
+ """Execute Censys v3 Platform queries based on target input type."""
257
+ if not self.is_configured():
258
+ logger.warning("Censys API credentials are not configured. Skipping Censys module.")
259
+ return []
260
+
261
+ # Skip if quota was previously exhausted
262
+ if self._quota_exhausted:
263
+ logger.info("Censys API quota exhausted. Skipping further Censys queries.")
264
+ return []
265
+
266
+ target = target.strip()
267
+
268
+ try:
269
+ # Clean prefix markers if present
270
+ clean_target = target
271
+ if clean_target.startswith("host:"):
272
+ clean_target = clean_target[5:]
273
+ elif clean_target.startswith("domain:"):
274
+ clean_target = clean_target[7:]
275
+
276
+ # 1. CIDR Network Check (e.g., 192.168.1.0/24) -> CenQL ip: 192.168.1.0/24
277
+ if "/" in clean_target:
278
+ try:
279
+ ipaddress.ip_network(clean_target, strict=False)
280
+ return await self.search_query(f"ip: {clean_target}")
281
+ except ValueError:
282
+ pass
283
+
284
+ # 2. Direct IP Address Check -> GET /v3/global/asset/host/{ip}
285
+ try:
286
+ ipaddress.ip_address(clean_target)
287
+ return await self.get_host_info(clean_target)
288
+ except ValueError:
289
+ pass
290
+
291
+ # 3. Domain Check -> CenQL names: example.com
292
+ if target.startswith("domain:") or ("." in target and " " not in target and ":" not in target):
293
+ domain_name = clean_target
294
+ return await self.search_query(f"names: {domain_name}")
295
+
296
+ # 4. Search Query / CenQL Query
297
+ query = clean_target
298
+ return await self.search_query(query)
299
+
300
+ except CensysQuotaExhaustedError as e:
301
+ print(f"⚠️ Censys API quota/balance exhausted. Skipping further Censys queries.")
302
+ self._quota_exhausted = True # Set flag to skip future calls
303
+ return []
304
+ except CensysRateLimitError as e:
305
+ logger.warning(f"Censys API rate limit exceeded for target {target}")
306
+ print(f"⚠️ Censys API rate limit exceeded. Please wait before making more requests or upgrade your plan.")
307
+ return []
308
+ except CensysAuthError as e:
309
+ logger.error(f"Censys authentication error for target {target}")
310
+ print(f"❌ Censys authentication failed. Please check your API credentials.")
311
+ return []
312
+ except CensysAPIError as e:
313
+ # Don't log/print if it's actually a quota exhaustion that wasn't caught properly
314
+ if "insufficient balance" not in str(e).lower():
315
+ logger.error(f"Censys API error for target {target}: {e}")
316
+ print(f"⚠️ Censys API error: {e}")
317
+ return []
318
+ except Exception as e:
319
+ logger.error(f"Censys module error for target {target}: {e}")
320
+ return []
321
+
322
+ async def get_host_info(self, ip: str) -> List[Finding]:
323
+ """Fetch complete host dossier and open services from Censys Platform API v3."""
324
+ if self._auth_failed or not self.is_configured():
325
+ return []
326
+
327
+ url = f"{self.base_url}/global/asset/host/{ip}"
328
+ headers = self._get_auth_headers(accept_header="application/vnd.censys.api.v3.host.v1+json")
329
+
330
+ try:
331
+ resp = await self.http_client.get(url=url, headers=headers, timeout=20.0, raise_for_status=False)
332
+ if resp.status_code == 200:
333
+ data = resp.json()
334
+ elif resp.status_code == 404:
335
+ logger.debug(f"Host {ip} not found in Censys Platform.")
336
+ return []
337
+ elif resp.status_code in (401, 403):
338
+ self._auth_failed = True
339
+ logger.warning(f"Censys Authentication/Permission error ({resp.status_code}): Access credentials invalid. Censys module bypassed.")
340
+ return []
341
+ elif resp.status_code == 422:
342
+ # Check if it's a quota exhaustion error first
343
+ is_quota_exhausted = False
344
+ try:
345
+ error_data = resp.json()
346
+ if isinstance(error_data, dict) and "errors" in error_data:
347
+ for error in error_data.get("errors", []):
348
+ if isinstance(error, dict) and "insufficient balance" in error.get("message", "").lower():
349
+ is_quota_exhausted = True
350
+ raise CensysQuotaExhaustedError(f"API quota exhausted for IP {ip}")
351
+ except CensysQuotaExhaustedError:
352
+ raise # Re-raise quota exhaustion error
353
+ except Exception:
354
+ pass # If we can't parse the error, treat it as a regular validation error
355
+
356
+ # Only log if it's not a quota exhaustion error
357
+ if not is_quota_exhausted:
358
+ logger.error(f"Censys validation error (422) for host {ip}: {resp.text}")
359
+ raise CensysAPIError(f"Validation error for IP {ip}: {resp.text}")
360
+ elif resp.status_code == 429:
361
+ logger.warning(f"Censys Rate limit exceeded for host lookup: {ip}")
362
+ raise CensysRateLimitError(f"Rate limit exceeded for IP {ip}")
363
+ else:
364
+ logger.warning(f"Censys API returned HTTP {resp.status_code} for host {ip}")
365
+ raise CensysAPIError(f"API error for IP {ip}: HTTP {resp.status_code}")
366
+ except (CensysQuotaExhaustedError, CensysRateLimitError, CensysAuthError, CensysAPIError):
367
+ # Re-raise our custom exceptions
368
+ raise
369
+ except Exception as exc:
370
+ logger.warning(f"Failed to fetch Censys host info for {ip}: {exc}")
371
+ raise CensysAPIError(f"Network error for IP {ip}: {exc}")
372
+
373
+ if not data or not isinstance(data, dict):
374
+ return []
375
+
376
+ return self._parse_host_result(ip, data)
377
+
378
+ def _parse_host_result(self, ip: str, result_data: Dict[str, Any]) -> List[Finding]:
379
+ """Parse structured host result from Censys Platform API v3 into standard Finding objects."""
380
+ findings: List[Finding] = []
381
+
382
+ # Unpack result / resource wrappers if present
383
+ if isinstance(result_data, dict):
384
+ if "result" in result_data and isinstance(result_data["result"], dict):
385
+ result_data = result_data["result"]
386
+ if "resource" in result_data and isinstance(result_data["resource"], dict):
387
+ result_data = result_data["resource"]
388
+
389
+ # Location details
390
+ location = result_data.get("location", {})
391
+ country_name = location.get("country") or location.get("country_name")
392
+ country_code = location.get("country_code")
393
+ city = location.get("city")
394
+ region_code = location.get("province") or location.get("region_code")
395
+
396
+ # Autonomous System
397
+ as_info = result_data.get("autonomous_system", {})
398
+ asn_num = as_info.get("asn")
399
+ asn_str = f"AS{asn_num}" if asn_num else None
400
+ org = as_info.get("name") or as_info.get("description")
401
+ isp = as_info.get("description") or as_info.get("name")
402
+
403
+ # Operating System
404
+ os_info = result_data.get("operating_system", {})
405
+ os_name = os_info.get("product") or os_info.get("uniform_resource_identifier")
406
+ if not os_name and os_info.get("vendor"):
407
+ os_name = f"{os_info.get('vendor')} {os_info.get('version', '')}".strip()
408
+
409
+ # DNS Names & Hostnames
410
+ dns_info = result_data.get("dns", {})
411
+ dns_names: List[str] = dns_info.get("names", []) if isinstance(dns_info.get("names"), list) else []
412
+ reverse_dns = dns_info.get("reverse_dns", {})
413
+ rev_names: List[str] = []
414
+ if isinstance(reverse_dns, dict):
415
+ rev_names = reverse_dns.get("names", []) if isinstance(reverse_dns.get("names"), list) else []
416
+ elif isinstance(reverse_dns, list):
417
+ rev_names = reverse_dns
418
+
419
+ hostnames: List[str] = list(set(dns_names + rev_names))
420
+ domains: List[str] = []
421
+
422
+ for h in hostnames:
423
+ parts = h.split(".")
424
+ if len(parts) >= 2:
425
+ base_dom = ".".join(parts[-2:])
426
+ if base_dom not in domains:
427
+ domains.append(base_dom)
428
+
429
+ services = result_data.get("services", [])
430
+ port_numbers: List[int] = []
431
+ identified_cves: List[str] = []
432
+
433
+ # 1. Parse Open Ports & Services
434
+ for svc in services:
435
+ if not isinstance(svc, dict):
436
+ continue
437
+
438
+ port_num = svc.get("port")
439
+ if port_num is None:
440
+ continue
441
+
442
+ port_numbers.append(port_num)
443
+ transport = (svc.get("transport_protocol") or "tcp").lower()
444
+
445
+ # Software / Product details
446
+ software_list = svc.get("software", [])
447
+ product = None
448
+ version = None
449
+ if software_list and isinstance(software_list, list) and len(software_list) > 0:
450
+ sw_item = software_list[0]
451
+ if isinstance(sw_item, dict):
452
+ product = sw_item.get("product") or sw_item.get("vendor")
453
+ version = sw_item.get("version")
454
+
455
+ # Endpoints & HTTP banners / titles / server headers
456
+ endpoints = svc.get("endpoints", [])
457
+ html_titles: List[str] = []
458
+ server_headers: List[str] = []
459
+ cert_names: List[str] = []
460
+
461
+ if isinstance(endpoints, list):
462
+ for ep in endpoints:
463
+ if not isinstance(ep, dict):
464
+ continue
465
+ http_info = ep.get("http", {})
466
+ if isinstance(http_info, dict):
467
+ t = http_info.get("html_title")
468
+ if t:
469
+ html_titles.append(str(t).strip())
470
+ srv_list = http_info.get("headers", {}).get("Server", {}).get("headers", [])
471
+ if isinstance(srv_list, list):
472
+ server_headers.extend(srv_list)
473
+ tls_info = ep.get("tls", {})
474
+ if isinstance(tls_info, dict):
475
+ c_names = tls_info.get("certificate", {}).get("names", [])
476
+ if isinstance(c_names, list):
477
+ cert_names.extend(c_names)
478
+
479
+ # Direct TLS certificates
480
+ direct_tls = svc.get("tls", {})
481
+ if isinstance(direct_tls, dict):
482
+ c_names = direct_tls.get("certificate", {}).get("names", [])
483
+ if isinstance(c_names, list):
484
+ cert_names.extend(c_names)
485
+
486
+ # Direct HTTP banner/title
487
+ direct_http = svc.get("http", {})
488
+ if isinstance(direct_http, dict):
489
+ direct_title = direct_http.get("response", {}).get("html_title") or direct_http.get("html_title")
490
+ if direct_title:
491
+ html_titles.append(str(direct_title).strip())
492
+
493
+ # Banner selection
494
+ banner_text = svc.get("banner")
495
+ if not banner_text and html_titles:
496
+ banner_text = html_titles[0]
497
+ elif not banner_text and server_headers:
498
+ banner_text = server_headers[0]
499
+
500
+ # Service name heuristic
501
+ service_name = svc.get("service_name") or svc.get("extended_service_name")
502
+ if not service_name or service_name == "unknown":
503
+ if product:
504
+ service_name = product.upper()
505
+ elif html_titles or server_headers or port_num in (80, 8080, 3000, 8000, 8888):
506
+ service_name = "HTTP"
507
+ elif cert_names or port_num in (443, 8443):
508
+ service_name = "HTTPS"
509
+ elif port_num == 22:
510
+ service_name = "SSH"
511
+ elif port_num == 21:
512
+ service_name = "FTP"
513
+ elif port_num == 500:
514
+ service_name = "IKE"
515
+ elif port_num == 1701:
516
+ service_name = "L2TP"
517
+ elif port_num == 1723:
518
+ service_name = "PPTP"
519
+ elif port_num == 8291:
520
+ service_name = "Winbox"
521
+ elif port_num == 2000:
522
+ service_name = "Bandwidth-Test"
523
+ else:
524
+ service_name = "unknown"
525
+
526
+ # Web URL
527
+ is_http = (
528
+ service_name.lower() in ("http", "https")
529
+ or port_num in (80, 443, 3000, 8080, 8443)
530
+ or bool(html_titles or server_headers)
531
+ )
532
+ is_ssl = (
533
+ "tls" in svc
534
+ or service_name.lower() == "https"
535
+ or port_num in (443, 8443)
536
+ or bool(cert_names)
537
+ )
538
+
539
+ web_url = None
540
+ if is_http:
541
+ scheme = "https" if is_ssl else "http"
542
+ web_url = f"{scheme}://{ip}:{port_num}"
543
+
544
+ # Certificate Names (Subdomains / Domains discovery)
545
+ for cname in cert_names:
546
+ clean_cname = cname.lstrip("*.").lower()
547
+ if clean_cname and clean_cname not in hostnames:
548
+ hostnames.append(clean_cname)
549
+ parts = clean_cname.split(".")
550
+ if len(parts) >= 2:
551
+ base_dom = ".".join(parts[-2:])
552
+ if base_dom not in domains:
553
+ domains.append(base_dom)
554
+
555
+ port_obj = PortData(
556
+ port=port_num,
557
+ transport=transport,
558
+ service=service_name,
559
+ product=product,
560
+ version=version,
561
+ banner=str(banner_text).strip() if banner_text else None,
562
+ url=web_url,
563
+ ssl=is_ssl,
564
+ sources=["Censys"],
565
+ )
566
+
567
+ findings.append(
568
+ Finding(
569
+ type=FindingType.OPEN_PORT,
570
+ target=ip,
571
+ value=f"{ip}:{port_num}",
572
+ source="Censys Platform v3",
573
+ host_ip=ip,
574
+ port_info=port_obj,
575
+ metadata={
576
+ "transport": transport,
577
+ "service": service_name,
578
+ "product": product,
579
+ "version": version,
580
+ "server": server_headers[0] if server_headers else None,
581
+ "title": html_titles[0] if html_titles else None,
582
+ },
583
+ )
584
+ )
585
+
586
+ # Check for service vulnerabilities
587
+ for v in svc.get("vulnerabilities", []):
588
+ cve_id = v if isinstance(v, str) else (v.get("cve") or v.get("cve_id") or v.get("id"))
589
+ if cve_id and isinstance(cve_id, str) and cve_id.upper().startswith("CVE-"):
590
+ identified_cves.append(cve_id.upper())
591
+
592
+ # Check for host-level vulnerabilities
593
+ for v in result_data.get("vulnerabilities", []):
594
+ cve_id = v if isinstance(v, str) else (v.get("cve") or v.get("cve_id") or v.get("id"))
595
+ if cve_id and isinstance(cve_id, str) and cve_id.upper().startswith("CVE-"):
596
+ identified_cves.append(cve_id.upper())
597
+
598
+ coordinates = location.get("coordinates", {}) if isinstance(location.get("coordinates"), dict) else {}
599
+ latitude = coordinates.get("latitude")
600
+ longitude = coordinates.get("longitude")
601
+ postal_code = location.get("postal_code")
602
+
603
+ host_info = HostInfoData(
604
+ ip=ip,
605
+ hostnames=sorted(list(set(hostnames))),
606
+ domains=sorted(list(set(domains))),
607
+ org=org,
608
+ isp=isp,
609
+ asn=asn_str,
610
+ os=os_name,
611
+ country_name=country_name,
612
+ country_code=country_code,
613
+ city=city,
614
+ region_code=region_code,
615
+ postal_code=postal_code,
616
+ latitude=latitude,
617
+ longitude=longitude,
618
+ ports=sorted(list(set(port_numbers))),
619
+ vulns=sorted(list(set(identified_cves))),
620
+ )
621
+
622
+ # 2. Host Info Finding
623
+ findings.append(
624
+ Finding(
625
+ type=FindingType.HOST_INFO,
626
+ target=ip,
627
+ value=ip,
628
+ source="Censys Platform v3",
629
+ host_ip=ip,
630
+ host_info=host_info,
631
+ metadata={"provider": "Censys", "location": location, "autonomous_system": as_info},
632
+ )
633
+ )
634
+
635
+ # 3. Associated Domains & Hostnames Findings
636
+ for domain in sorted(list(set(domains))):
637
+ findings.append(
638
+ Finding(
639
+ type=FindingType.ASSOCIATED_DOMAIN,
640
+ target=ip,
641
+ value=domain,
642
+ source="Censys Platform v3 (Host Domains)",
643
+ host_ip=ip,
644
+ )
645
+ )
646
+ for hname in sorted(list(set(hostnames))):
647
+ findings.append(
648
+ Finding(
649
+ type=FindingType.SUBDOMAIN,
650
+ target=ip,
651
+ value=hname,
652
+ source="Censys Platform v3 (DNS/Certs)",
653
+ host_ip=ip,
654
+ )
655
+ )
656
+
657
+ # 4. Vulnerabilities Findings
658
+ for cve in sorted(list(set(identified_cves))):
659
+ findings.append(
660
+ Finding(
661
+ type=FindingType.VULNERABILITY,
662
+ target=ip,
663
+ value=cve,
664
+ source="Censys Platform v3",
665
+ host_ip=ip,
666
+ vulnerability=VulnerabilityData(cve_id=cve),
667
+ metadata={"ip": ip},
668
+ )
669
+ )
670
+
671
+ return findings
672
+
673
+ async def search_query(
674
+ self,
675
+ query: str,
676
+ page_size: int = 100,
677
+ max_pages: Optional[int] = None,
678
+ ) -> List[Finding]:
679
+ """Execute CenQL unified search via Censys Platform API v3 across all pages (POST /v3/global/search/query)."""
680
+ if self._auth_failed or not self.is_configured():
681
+ return []
682
+
683
+ findings: List[Finding] = []
684
+ url = f"{self.base_url}/global/search/query"
685
+ headers = self._get_auth_headers()
686
+ cursor: Optional[str] = None
687
+ page_count = 0
688
+
689
+ while True:
690
+ if max_pages is not None and page_count >= max_pages:
691
+ break
692
+ page_count += 1
693
+
694
+ payload: Dict[str, Any] = {
695
+ "query": query,
696
+ "page_size": page_size,
697
+ }
698
+ if cursor:
699
+ payload["cursor"] = cursor
700
+
701
+ try:
702
+ resp = await self.http_client.post(
703
+ url=url,
704
+ headers=headers,
705
+ json=payload,
706
+ timeout=25.0,
707
+ raise_for_status=False,
708
+ )
709
+
710
+ if resp.status_code == 200:
711
+ data = resp.json()
712
+ elif resp.status_code == 403:
713
+ err_msg = resp.text
714
+ if "organization ID for API access" in err_msg or "Free users" in err_msg:
715
+ logger.info(
716
+ "Censys CenQL search queries require an Organization ID / API Access tier. "
717
+ "Direct IP host lookups are supported on Free tier."
718
+ )
719
+ else:
720
+ self._auth_failed = True
721
+ logger.warning(f"Censys Authentication/Permission error (403): Access credentials invalid or unauthorized. Censys module bypassed.")
722
+ break
723
+ elif resp.status_code == 401:
724
+ self._auth_failed = True
725
+ logger.warning("Censys Authentication error (401): Access credentials invalid. Censys module bypassed.")
726
+ break
727
+ elif resp.status_code == 422:
728
+ # Check if it's a quota exhaustion error first
729
+ is_quota_exhausted = False
730
+ try:
731
+ error_data = resp.json()
732
+ if isinstance(error_data, dict) and "errors" in error_data:
733
+ for error in error_data.get("errors", []):
734
+ if isinstance(error, dict) and "insufficient balance" in error.get("message", "").lower():
735
+ is_quota_exhausted = True
736
+ raise CensysQuotaExhaustedError(f"API quota exhausted for query '{query}'")
737
+ except CensysQuotaExhaustedError:
738
+ raise # Re-raise quota exhaustion error
739
+ except Exception:
740
+ pass # If we can't parse the error, treat it as a regular validation error
741
+
742
+ # Only log if it's not a quota exhaustion error
743
+ if not is_quota_exhausted:
744
+ logger.error(f"Censys CenQL Validation Error (422): {resp.text}")
745
+ raise CensysAPIError(f"Query validation error for '{query}': {resp.text}")
746
+ elif resp.status_code == 429:
747
+ logger.warning("Censys Rate limit exceeded during search query.")
748
+ raise CensysRateLimitError(f"Rate limit exceeded for query '{query}'")
749
+ else:
750
+ logger.warning(f"Censys search query returned HTTP {resp.status_code}: {resp.text}")
751
+ raise CensysAPIError(f"API error for query '{query}': HTTP {resp.status_code}")
752
+ except (CensysQuotaExhaustedError, CensysRateLimitError, CensysAuthError, CensysAPIError):
753
+ # Re-raise our custom exceptions
754
+ raise
755
+ except Exception as exc:
756
+ logger.warning(f"Error executing Censys search query '{query}': {exc}")
757
+ raise CensysAPIError(f"Network error for query '{query}': {exc}")
758
+
759
+ if not data or not isinstance(data, dict):
760
+ break
761
+
762
+ result_data = data.get("result", {})
763
+ hits = result_data.get("hits", [])
764
+ if not hits:
765
+ break
766
+
767
+ for hit in hits:
768
+ ip_str = hit.get("ip")
769
+ if not ip_str:
770
+ continue
771
+
772
+ if "services" in hit and hit.get("services"):
773
+ host_findings = self._parse_host_result(ip_str, hit)
774
+ else:
775
+ host_findings = await self.get_host_info(ip_str)
776
+
777
+ findings.extend(host_findings)
778
+
779
+ links = result_data.get("links", {})
780
+ cursor = links.get("next") or result_data.get("cursor")
781
+ if not cursor:
782
+ break
783
+
784
+ return findings
785
+
786
+ async def aggregate_search(
787
+ self,
788
+ query: str,
789
+ field: str,
790
+ num_buckets: int = 10,
791
+ ) -> Dict[str, Any]:
792
+ """Aggregate search results across global assets (POST /v3/global/search/aggregate)."""
793
+ url = f"{self.base_url}/global/search/aggregate"
794
+ headers = self._get_auth_headers()
795
+ payload = {
796
+ "query": query,
797
+ "field": field,
798
+ "num_buckets": num_buckets,
799
+ }
800
+ res = await self.http_client.post_json(url=url, headers=headers, json=payload, timeout=20.0)
801
+ return res if isinstance(res, dict) else {}
802
+
803
+ async def get_certificate(self, fingerprint: str) -> Dict[str, Any]:
804
+ """Fetch SSL/TLS certificate details by SHA-256 fingerprint (GET /v3/global/asset/certificate/{fingerprint})."""
805
+ url = f"{self.base_url}/global/asset/certificate/{fingerprint}"
806
+ headers = self._get_auth_headers()
807
+ res = await self.http_client.get_json(url=url, headers=headers, timeout=20.0)
808
+ return res if isinstance(res, dict) else {}
809
+
810
+ def convert_legacy_query(self, legacy_query: str) -> Dict[str, Any]:
811
+ """Convert legacy search query syntax (v1/v2) to modern CenQL (POST /v3/global/search/convert)."""
812
+ client = CensysPlatformClient(pat_token=self.pat_token, org_id=self.org_id)
813
+ return client.convert_legacy_query(legacy_query)