secator 0.5.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of secator might be problematic. Click here for more details.

Files changed (84) hide show
  1. secator/celery.py +160 -185
  2. secator/celery_utils.py +268 -0
  3. secator/cli.py +327 -106
  4. secator/config.py +27 -11
  5. secator/configs/workflows/host_recon.yaml +5 -3
  6. secator/configs/workflows/port_scan.yaml +7 -3
  7. secator/configs/workflows/url_bypass.yaml +10 -0
  8. secator/configs/workflows/url_vuln.yaml +1 -1
  9. secator/decorators.py +169 -92
  10. secator/definitions.py +10 -3
  11. secator/exporters/__init__.py +7 -5
  12. secator/exporters/console.py +10 -0
  13. secator/exporters/csv.py +27 -19
  14. secator/exporters/gdrive.py +16 -11
  15. secator/exporters/json.py +3 -1
  16. secator/exporters/table.py +30 -2
  17. secator/exporters/txt.py +20 -16
  18. secator/hooks/gcs.py +53 -0
  19. secator/hooks/mongodb.py +54 -28
  20. secator/output_types/__init__.py +29 -11
  21. secator/output_types/_base.py +11 -1
  22. secator/output_types/error.py +36 -0
  23. secator/output_types/exploit.py +1 -1
  24. secator/output_types/info.py +24 -0
  25. secator/output_types/ip.py +7 -0
  26. secator/output_types/port.py +8 -1
  27. secator/output_types/progress.py +6 -1
  28. secator/output_types/record.py +3 -1
  29. secator/output_types/stat.py +33 -0
  30. secator/output_types/tag.py +6 -4
  31. secator/output_types/url.py +6 -3
  32. secator/output_types/vulnerability.py +3 -2
  33. secator/output_types/warning.py +24 -0
  34. secator/report.py +55 -23
  35. secator/rich.py +44 -39
  36. secator/runners/_base.py +622 -635
  37. secator/runners/_helpers.py +5 -91
  38. secator/runners/celery.py +18 -0
  39. secator/runners/command.py +364 -211
  40. secator/runners/scan.py +8 -24
  41. secator/runners/task.py +21 -55
  42. secator/runners/workflow.py +41 -40
  43. secator/scans/__init__.py +28 -0
  44. secator/serializers/dataclass.py +6 -0
  45. secator/serializers/json.py +10 -5
  46. secator/serializers/regex.py +12 -4
  47. secator/tasks/_categories.py +6 -3
  48. secator/tasks/bbot.py +293 -0
  49. secator/tasks/bup.py +98 -0
  50. secator/tasks/cariddi.py +38 -49
  51. secator/tasks/dalfox.py +3 -0
  52. secator/tasks/dirsearch.py +12 -23
  53. secator/tasks/dnsx.py +49 -30
  54. secator/tasks/dnsxbrute.py +2 -0
  55. secator/tasks/feroxbuster.py +8 -17
  56. secator/tasks/ffuf.py +3 -2
  57. secator/tasks/fping.py +3 -3
  58. secator/tasks/gau.py +5 -0
  59. secator/tasks/gf.py +2 -2
  60. secator/tasks/gospider.py +4 -0
  61. secator/tasks/grype.py +9 -9
  62. secator/tasks/h8mail.py +31 -41
  63. secator/tasks/httpx.py +58 -21
  64. secator/tasks/katana.py +18 -22
  65. secator/tasks/maigret.py +26 -24
  66. secator/tasks/mapcidr.py +2 -3
  67. secator/tasks/msfconsole.py +4 -16
  68. secator/tasks/naabu.py +3 -1
  69. secator/tasks/nmap.py +50 -35
  70. secator/tasks/nuclei.py +9 -2
  71. secator/tasks/searchsploit.py +17 -9
  72. secator/tasks/subfinder.py +5 -1
  73. secator/tasks/wpscan.py +79 -93
  74. secator/template.py +61 -45
  75. secator/thread.py +24 -0
  76. secator/utils.py +330 -80
  77. secator/utils_test.py +48 -23
  78. secator/workflows/__init__.py +28 -0
  79. {secator-0.5.2.dist-info → secator-0.7.0.dist-info}/METADATA +12 -6
  80. secator-0.7.0.dist-info/RECORD +115 -0
  81. {secator-0.5.2.dist-info → secator-0.7.0.dist-info}/WHEEL +1 -1
  82. secator-0.5.2.dist-info/RECORD +0 -101
  83. {secator-0.5.2.dist-info → secator-0.7.0.dist-info}/entry_points.txt +0 -0
  84. {secator-0.5.2.dist-info → secator-0.7.0.dist-info}/licenses/LICENSE +0 -0
secator/config.py CHANGED
@@ -20,6 +20,7 @@ StrExpandHome = Annotated[str, AfterValidator(lambda v: v.replace('~', str(Path.
20
20
  ROOT_FOLDER = Path(__file__).parent.parent
21
21
  LIB_FOLDER = ROOT_FOLDER / 'secator'
22
22
  CONFIGS_FOLDER = LIB_FOLDER / 'configs'
23
+ DATA_FOLDER = os.environ.get('SECATOR_DIRS_DATA') or str(Path.home() / '.secator')
23
24
 
24
25
 
25
26
  class StrictModel(BaseModel, extra='forbid'):
@@ -28,12 +29,13 @@ class StrictModel(BaseModel, extra='forbid'):
28
29
 
29
30
  class Directories(StrictModel):
30
31
  bin: Directory = Path.home() / '.local' / 'bin'
31
- data: Directory = Path.home() / '.secator'
32
+ data: Directory = Path(DATA_FOLDER)
32
33
  templates: Directory = ''
33
34
  reports: Directory = ''
34
35
  wordlists: Directory = ''
35
36
  cves: Directory = ''
36
37
  payloads: Directory = ''
38
+ performance: Directory = ''
37
39
  revshells: Directory = ''
38
40
  celery: Directory = ''
39
41
  celery_data: Directory = ''
@@ -42,7 +44,7 @@ class Directories(StrictModel):
42
44
  @model_validator(mode='after')
43
45
  def set_default_folders(self) -> Self:
44
46
  """Set folders to be relative to the data folders if they are unspecified in config."""
45
- for folder in ['templates', 'reports', 'wordlists', 'cves', 'payloads', 'revshells', 'celery', 'celery_data', 'celery_results']: # noqa: E501
47
+ for folder in ['templates', 'reports', 'wordlists', 'cves', 'payloads', 'performance', 'revshells', 'celery', 'celery_data', 'celery_results']: # noqa: E501
46
48
  rel_target = '/'.join(folder.split('_'))
47
49
  val = getattr(self, folder) or self.data / rel_target
48
50
  setattr(self, folder, val)
@@ -61,6 +63,7 @@ class Celery(StrictModel):
61
63
  broker_visibility_timeout: int = 3600
62
64
  override_default_logging: bool = True
63
65
  result_backend: StrExpandHome = ''
66
+ result_expires: int = 86400 # 1 day
64
67
 
65
68
 
66
69
  class Cli(StrictModel):
@@ -70,8 +73,11 @@ class Cli(StrictModel):
70
73
 
71
74
 
72
75
  class Runners(StrictModel):
73
- input_chunk_size: int = 1000
74
- progress_update_frequency: int = 60
76
+ input_chunk_size: int = 100
77
+ progress_update_frequency: int = 20
78
+ stat_update_frequency: int = 20
79
+ backend_update_frequency: int = 5
80
+ poll_frequency: int = 5
75
81
  skip_cve_search: bool = False
76
82
  skip_cve_low_confidence: bool = True
77
83
  remove_duplicates: bool = False
@@ -81,20 +87,21 @@ class HTTP(StrictModel):
81
87
  socks5_proxy: str = 'socks5://127.0.0.1:9050'
82
88
  http_proxy: str = 'https://127.0.0.1:9080'
83
89
  store_responses: bool = False
90
+ response_max_size_bytes: int = 100000 # 100MB
84
91
  proxychains_command: str = 'proxychains'
85
92
  freeproxy_timeout: int = 1
86
93
 
87
94
 
88
95
  class Tasks(StrictModel):
89
- exporters: List[str] = ['json', 'csv']
96
+ exporters: List[str] = ['json', 'csv', 'txt']
90
97
 
91
98
 
92
99
  class Workflows(StrictModel):
93
- exporters: List[str] = ['json', 'csv']
100
+ exporters: List[str] = ['json', 'csv', 'txt']
94
101
 
95
102
 
96
103
  class Scans(StrictModel):
97
- exporters: List[str] = ['json', 'csv']
104
+ exporters: List[str] = ['json', 'csv', 'txt']
98
105
 
99
106
 
100
107
  class Payloads(StrictModel):
@@ -114,12 +121,18 @@ class Wordlists(StrictModel):
114
121
  lists: Dict[str, List[str]] = {}
115
122
 
116
123
 
117
- class GoogleAddon(StrictModel):
124
+ class GoogleDriveAddon(StrictModel):
118
125
  enabled: bool = False
119
126
  drive_parent_folder_id: str = ''
120
127
  credentials_path: str = ''
121
128
 
122
129
 
130
+ class GoogleCloudStorageAddon(StrictModel):
131
+ enabled: bool = False
132
+ bucket_name: str = ''
133
+ credentials_path: str = ''
134
+
135
+
123
136
  class WorkerAddon(StrictModel):
124
137
  enabled: bool = False
125
138
 
@@ -128,10 +141,13 @@ class MongodbAddon(StrictModel):
128
141
  enabled: bool = False
129
142
  url: str = 'mongodb://localhost'
130
143
  update_frequency: int = 60
144
+ max_pool_size: int = 10
145
+ server_selection_timeout_ms: int = 5000
131
146
 
132
147
 
133
148
  class Addons(StrictModel):
134
- google: GoogleAddon = GoogleAddon()
149
+ gdrive: GoogleDriveAddon = GoogleDriveAddon()
150
+ gcs: GoogleCloudStorageAddon = GoogleCloudStorageAddon()
135
151
  worker: WorkerAddon = WorkerAddon()
136
152
  mongodb: MongodbAddon = MongodbAddon()
137
153
 
@@ -161,7 +177,7 @@ class Config(DotMap):
161
177
  >>> config = Config.parse(path='/path/to/config.yml') # get custom config (from YAML file).
162
178
  >>> config.print() # print config without defaults.
163
179
  >>> config.print(partial=False) # print full config.
164
- >>> config.set('addons.google.enabled', False) # set value in config.
180
+ >>> config.set('addons.gdrive.enabled', False) # set value in config.
165
181
  >>> config.save() # save config back to disk.
166
182
  """
167
183
 
@@ -512,7 +528,7 @@ def download_files(data: dict, target_folder: Path, offline_mode: bool, type: st
512
528
  else:
513
529
  # Download file from URL
514
530
  ext = url_or_path.split('.')[-1]
515
- filename = f'{name}.{ext}'
531
+ filename = f'{name}.{ext}' if not name.endswith(ext) else name
516
532
  target_path = target_folder / filename
517
533
  if not target_path.exists():
518
534
  try:
@@ -11,6 +11,8 @@ tasks:
11
11
  description: Find open ports
12
12
  nmap:
13
13
  description: Search for vulnerabilities on open ports
14
+ skip_host_discovery: True
15
+ version_detection: True
14
16
  targets_: port.host
15
17
  ports_: port.port
16
18
  httpx:
@@ -18,7 +20,7 @@ tasks:
18
20
  targets_:
19
21
  - type: port
20
22
  field: '{host}:{port}'
21
- condition: item._source == 'nmap'
23
+ condition: item._source.startswith('nmap')
22
24
  _group:
23
25
  nuclei/network:
24
26
  description: Scan network and SSL vulnerabilities
@@ -32,10 +34,10 @@ tasks:
32
34
  condition: item.status_code != 0
33
35
  results:
34
36
  - type: port
35
- condition: item._source == 'nmap'
37
+ condition: item._source.startswith('nmap')
36
38
 
37
39
  - type: vulnerability
38
40
  # condition: item.confidence == 'high'
39
41
 
40
42
  - type: url
41
- condition: item.status_code != 0
43
+ condition: item.status_code != 0
@@ -5,11 +5,15 @@ description: Port scan
5
5
  tags: [recon, network, http, vuln]
6
6
  input_types:
7
7
  - host
8
+ - cidr_range
8
9
  tasks:
9
10
  naabu:
10
11
  description: Find open ports
12
+ ports: "-" # scan all ports
11
13
  nmap:
12
14
  description: Search for vulnerabilities on open ports
15
+ skip_host_discovery: True
16
+ version_detection: True
13
17
  targets_: port.host
14
18
  ports_: port.port
15
19
  _group:
@@ -18,17 +22,17 @@ tasks:
18
22
  targets_:
19
23
  - type: port
20
24
  field: '{host}~{service_name}'
21
- condition: item._source == 'nmap' and len(item.service_name.split('/')) > 1
25
+ condition: item._source.startswith('nmap') and len(item.service_name.split('/')) > 1
22
26
  httpx:
23
27
  description: Probe HTTP services on open ports
24
28
  targets_:
25
29
  - type: port
26
30
  field: '{host}:{port}'
27
- condition: item._source == 'nmap'
31
+ condition: item._source.startswith('nmap')
28
32
  results:
29
33
  - type: port
30
34
 
31
35
  - type: url
32
36
  condition: item.status_code != 0
33
37
 
34
- - type: vulnerability
38
+ - type: vulnerability
@@ -0,0 +1,10 @@
1
+ type: workflow
2
+ name: url_bypass
3
+ alias: urlbypass
4
+ description: Try bypass techniques for 4xx URLs
5
+ tags: [http, crawl]
6
+ input_types:
7
+ - url
8
+ tasks:
9
+ bup:
10
+ description: Bypass 4xx
@@ -34,7 +34,7 @@ tasks:
34
34
  targets_:
35
35
  - type: tag
36
36
  field: match
37
- condition: item._source == "gf"
37
+ condition: item._source.startswith("gf")
38
38
 
39
39
  # TODO: Add support for SQLMap
40
40
  # sqlmap:
secator/decorators.py CHANGED
@@ -1,30 +1,33 @@
1
1
  import sys
2
+
2
3
  from collections import OrderedDict
3
4
 
4
5
  import rich_click as click
5
6
  from rich_click.rich_click import _get_rich_console
6
7
  from rich_click.rich_group import RichGroup
7
8
 
8
- from secator.definitions import ADDONS_ENABLED, OPT_NOT_SUPPORTED
9
9
  from secator.config import CONFIG
10
+ from secator.definitions import ADDONS_ENABLED, OPT_NOT_SUPPORTED
10
11
  from secator.runners import Scan, Task, Workflow
11
- from secator.utils import (deduplicate, expand_input, get_command_category,
12
- get_command_cls)
12
+ from secator.utils import (deduplicate, expand_input, get_command_category)
13
13
 
14
14
  RUNNER_OPTS = {
15
15
  'output': {'type': str, 'default': None, 'help': 'Output options (-o table,json,csv,gdrive)', 'short': 'o'},
16
16
  'workspace': {'type': str, 'default': 'default', 'help': 'Workspace', 'short': 'ws'},
17
- 'json': {'is_flag': True, 'default': False, 'help': 'Enable JSON mode'},
18
- 'orig': {'is_flag': True, 'default': False, 'help': 'Enable original output (no schema conversion)'},
19
- 'raw': {'is_flag': True, 'default': False, 'help': 'Enable text output for piping to other tools'},
17
+ 'print_json': {'is_flag': True, 'short': 'json', 'default': False, 'help': 'Print items as JSON lines'},
18
+ 'print_raw': {'is_flag': True, 'short': 'raw', 'default': False, 'help': 'Print items in raw format'},
19
+ 'print_stat': {'is_flag': True, 'short': 'stat', 'default': False, 'help': 'Print runtime statistics'},
20
+ 'print_format': {'default': '', 'short': 'fmt', 'help': 'Output formatting string'},
21
+ 'enable_profiler': {'is_flag': True, 'short': 'prof', 'default': False, 'help': 'Enable runner profiling'},
20
22
  'show': {'is_flag': True, 'default': False, 'help': 'Show command that will be run (tasks only)'},
21
- 'format': {'default': '', 'short': 'fmt', 'help': 'Output formatting string'},
23
+ 'no_process': {'is_flag': True, 'default': False, 'help': 'Disable secator processing'},
22
24
  # 'filter': {'default': '', 'short': 'f', 'help': 'Results filter', 'short': 'of'}, # TODO add this
23
25
  'quiet': {'is_flag': True, 'default': False, 'help': 'Enable quiet mode'},
24
26
  }
25
27
 
26
28
  RUNNER_GLOBAL_OPTS = {
27
29
  'sync': {'is_flag': True, 'help': 'Run tasks synchronously (automatic if no worker is alive)'},
30
+ 'worker': {'is_flag': True, 'default': False, 'help': 'Run tasks in worker'},
28
31
  'proxy': {'type': str, 'help': 'HTTP proxy'},
29
32
  'driver': {'type': str, 'help': 'Export real-time results. E.g: "mongodb"'}
30
33
  # 'debug': {'type': int, 'default': 0, 'help': 'Debug mode'},
@@ -100,20 +103,34 @@ class OrderedGroup(RichGroup):
100
103
  return self.commands
101
104
 
102
105
 
103
- def get_command_options(*tasks):
104
- """Get unified list of command options from a list of secator tasks classes.
106
+ def get_command_options(config):
107
+ """Get unified list of command options from a list of secator tasks classes and optionally a Runner config.
105
108
 
106
109
  Args:
107
- tasks (list): List of secator command classes.
110
+ config (TemplateLoader): Current runner config.
108
111
 
109
112
  Returns:
110
113
  list: List of deduplicated options.
111
114
  """
115
+ from secator.utils import debug
112
116
  opt_cache = []
113
117
  all_opts = OrderedDict({})
118
+ tasks = config.flat_tasks
119
+ tasks_cls = set([c['class'] for c in tasks.values()])
114
120
 
115
- for cls in tasks:
121
+ # Loop through tasks and set options
122
+ for cls in tasks_cls:
116
123
  opts = OrderedDict(RUNNER_GLOBAL_OPTS, **RUNNER_OPTS, **cls.meta_opts, **cls.opts)
124
+
125
+ # Find opts defined in config corresponding to this task class
126
+ # TODO: rework this as this ignores subsequent tasks of the same task class
127
+ task_config_opts = {}
128
+ if config.type != 'task':
129
+ for k, v in tasks.items():
130
+ if v['class'] == cls:
131
+ task_config_opts = v['opts']
132
+
133
+ # Loop through options
117
134
  for opt, opt_conf in opts.items():
118
135
 
119
136
  # Get opt key map if any
@@ -126,6 +143,7 @@ def get_command_options(*tasks):
126
143
  and opt not in RUNNER_GLOBAL_OPTS:
127
144
  continue
128
145
 
146
+ # Opt is defined as unsupported
129
147
  if opt_key_map.get(opt) == OPT_NOT_SUPPORTED:
130
148
  continue
131
149
 
@@ -143,17 +161,50 @@ def get_command_options(*tasks):
143
161
  elif opt in RUNNER_GLOBAL_OPTS:
144
162
  prefix = 'Execution'
145
163
 
164
+ # Get opt conf
165
+ conf = opt_conf.copy()
166
+ conf['show_default'] = True
167
+ conf['prefix'] = prefix
168
+ opt_default = conf.get('default', None)
169
+ opt_is_flag = conf.get('is_flag', False)
170
+ opt_value_in_config = task_config_opts.get(opt)
171
+
172
+ # Check if opt already defined in config
173
+ if opt_value_in_config:
174
+ if conf.get('required', False):
175
+ debug('OPT (skipped: opt is required and defined in config)', obj={'opt': opt}, sub=f'cli.{config.name}', verbose=True) # noqa: E501
176
+ continue
177
+ mapped_value = cls.opt_value_map.get(opt)
178
+ if callable(mapped_value):
179
+ opt_value_in_config = mapped_value(opt_value_in_config)
180
+ elif mapped_value:
181
+ opt_value_in_config = mapped_value
182
+ if opt_value_in_config != opt_default:
183
+ if opt in opt_cache:
184
+ continue
185
+ if opt_is_flag:
186
+ conf['reverse'] = True
187
+ conf['default'] = not conf['default']
188
+ # print(f'{opt}: change default to {opt_value_in_config}')
189
+ conf['default'] = opt_value_in_config
190
+
191
+ # If opt is a flag but the default is True, add opposite flag
192
+ if opt_is_flag and opt_default is True:
193
+ conf['reverse'] = True
194
+
146
195
  # Check if opt already processed before
147
- opt = opt.replace('_', '-')
148
196
  if opt in opt_cache:
197
+ # debug('OPT (skipped: opt is already in opt cache)', obj={'opt': opt}, sub=f'cli.{config.name}', verbose=True)
149
198
  continue
150
199
 
151
200
  # Build help
152
- conf = opt_conf.copy()
153
- conf['show_default'] = True
154
- conf['prefix'] = prefix
155
- all_opts[opt] = conf
156
201
  opt_cache.append(opt)
202
+ opt = opt.replace('_', '-')
203
+ all_opts[opt] = conf
204
+
205
+ # Debug
206
+ debug_conf = OrderedDict({'opt': opt, 'config_val': opt_value_in_config or 'N/A', **conf.copy()})
207
+ debug('OPT', obj=debug_conf, sub=f'cli.{config.name}', verbose=True)
157
208
 
158
209
  return all_opts
159
210
 
@@ -171,11 +222,18 @@ def decorate_command_options(opts):
171
222
  reversed_opts = OrderedDict(list(opts.items())[::-1])
172
223
  for opt_name, opt_conf in reversed_opts.items():
173
224
  conf = opt_conf.copy()
174
- short = conf.pop('short', None)
175
- conf.pop('internal', False)
225
+ short_opt = conf.pop('short', None)
226
+ conf.pop('internal', None)
176
227
  conf.pop('prefix', None)
228
+ conf.pop('shlex', None)
229
+ conf.pop('meta', None)
230
+ conf.pop('supported', None)
231
+ reverse = conf.pop('reverse', False)
177
232
  long = f'--{opt_name}'
178
- short = f'-{short}' if short else f'-{opt_name}'
233
+ short = f'-{short_opt}' if short_opt else f'-{opt_name}'
234
+ if reverse:
235
+ long += f'/--no-{opt_name}'
236
+ short += f'/-n{short_opt}' if short else f'/-n{opt_name}'
179
237
  f = click.option(long, short, **conf)(f)
180
238
  return f
181
239
  return decorator
@@ -188,66 +246,56 @@ def task():
188
246
  return decorator
189
247
 
190
248
 
249
+ def generate_cli_subcommand(cli_endpoint, func, **opts):
250
+ return cli_endpoint.command(**opts)(func)
251
+
252
+
191
253
  def register_runner(cli_endpoint, config):
192
- fmt_opts = {
193
- 'print_cmd': True,
194
- }
195
- short_help = ''
196
- input_type = 'targets'
254
+ name = config.name
197
255
  input_required = True
198
- runner_cls = None
199
- tasks = []
200
- no_args_is_help = True
256
+ input_type = 'targets'
257
+ command_opts = {
258
+ 'no_args_is_help': True,
259
+ 'context_settings': {
260
+ 'ignore_unknown_options': False,
261
+ 'allow_extra_args': False
262
+ }
263
+ }
201
264
 
202
265
  if cli_endpoint.name == 'scan':
203
- # TODO: this should be refactored to scan.get_tasks_from_conf() or scan.tasks
204
- from secator.cli import ALL_CONFIGS
205
- tasks = [
206
- get_command_cls(task)
207
- for workflow in ALL_CONFIGS.workflow
208
- for task in Task.get_tasks_from_conf(workflow.tasks)
209
- if workflow.name in list(config.workflows.keys())
210
- ]
211
- input_type = 'targets'
212
- name = config.name
213
- short_help = config.description or ''
214
- if config.alias:
215
- short_help += f' [dim]alias: {config.alias}'
216
- fmt_opts['print_start'] = True
217
- fmt_opts['print_run_summary'] = True
218
- fmt_opts['print_progress'] = False
219
266
  runner_cls = Scan
267
+ short_help = config.description or ''
268
+ short_help += f' [dim]alias: {config.alias}' if config.alias else ''
269
+ command_opts.update({
270
+ 'name': name,
271
+ 'short_help': short_help
272
+ })
220
273
 
221
274
  elif cli_endpoint.name == 'workflow':
222
- # TODO: this should be refactored to workflow.get_tasks_from_conf() or workflow.tasks
223
- tasks = [
224
- get_command_cls(task) for task in Task.get_tasks_from_conf(config.tasks)
225
- ]
226
- input_type = 'targets'
227
- name = config.name
228
- short_help = config.description or ''
229
- if config.alias:
230
- short_help = f'{short_help:<55} [dim](alias)[/][bold cyan] {config.alias}'
231
- fmt_opts['print_start'] = True
232
- fmt_opts['print_run_summary'] = True
233
- fmt_opts['print_progress'] = False
234
275
  runner_cls = Workflow
276
+ short_help = config.description or ''
277
+ short_help = f'{short_help:<55} [dim](alias)[/][bold cyan] {config.alias}' if config.alias else ''
278
+ command_opts.update({
279
+ 'name': name,
280
+ 'short_help': short_help
281
+ })
235
282
 
236
283
  elif cli_endpoint.name == 'task':
237
- tasks = [
238
- get_command_cls(config.name)
239
- ]
284
+ runner_cls = Task
285
+ input_required = False # allow targets from stdin
240
286
  task_cls = Task.get_task_class(config.name)
241
287
  task_category = get_command_category(task_cls)
242
288
  input_type = task_cls.input_type or 'targets'
243
- name = config.name
244
289
  short_help = f'[magenta]{task_category:<15}[/]{task_cls.__doc__}'
245
- fmt_opts['print_item_count'] = True
246
- runner_cls = Task
247
- no_args_is_help = False
248
- input_required = False
290
+ command_opts.update({
291
+ 'name': name,
292
+ 'short_help': short_help,
293
+ 'no_args_is_help': False
294
+ })
249
295
 
250
- options = get_command_options(*tasks)
296
+ else:
297
+ raise ValueError(f"Unrecognized runner endpoint name {cli_endpoint.name}")
298
+ options = get_command_options(config)
251
299
 
252
300
  # TODO: maybe allow this in the future
253
301
  # def get_unknown_opts(ctx):
@@ -262,24 +310,59 @@ def register_runner(cli_endpoint, config):
262
310
  @decorate_command_options(options)
263
311
  @click.pass_context
264
312
  def func(ctx, **opts):
265
- opts.update(fmt_opts)
266
313
  sync = opts['sync']
267
- # debug = opts['debug']
314
+ worker = opts.pop('worker')
268
315
  ws = opts.pop('workspace')
269
316
  driver = opts.pop('driver', '')
270
317
  show = opts['show']
271
318
  context = {'workspace_name': ws}
319
+
320
+ # Remove options whose values are default values
321
+ for k, v in options.items():
322
+ opt_name = k.replace('-', '_')
323
+ if opt_name in opts and opts[opt_name] == v.get('default', None):
324
+ del opts[opt_name]
325
+
272
326
  # TODO: maybe allow this in the future
273
327
  # unknown_opts = get_unknown_opts(ctx)
274
328
  # opts.update(unknown_opts)
275
- targets = opts.pop(input_type)
276
- targets = expand_input(targets)
329
+
330
+ # Expand input
331
+ inputs = opts.pop(input_type)
332
+ inputs = expand_input(inputs, ctx)
333
+
334
+ # Build hooks from driver name
335
+ hooks = []
336
+ drivers = driver.split(',') if driver else []
337
+ console = _get_rich_console()
338
+ supported_drivers = ['mongodb', 'gcs']
339
+ for driver in drivers:
340
+ if driver in supported_drivers:
341
+ if not ADDONS_ENABLED[driver]:
342
+ console.print(f'[bold red]Missing "{driver}" addon: please run `secator install addons {driver}`[/].')
343
+ sys.exit(1)
344
+ from secator.utils import import_dynamic
345
+ driver_hooks = import_dynamic(f'secator.hooks.{driver}', 'HOOKS')
346
+ if driver_hooks is None:
347
+ console.print(f'[bold red]Missing "secator.hooks.{driver}.HOOKS".[/]')
348
+ sys.exit(1)
349
+ hooks.append(driver_hooks)
350
+ else:
351
+ supported_drivers_str = ', '.join([f'[bold green]{_}[/]' for _ in supported_drivers])
352
+ console.print(f'[bold red]Driver "{driver}" is not supported.[/]')
353
+ console.print(f'Supported drivers: {supported_drivers_str}')
354
+ sys.exit(1)
355
+
356
+ from secator.utils import deep_merge_dicts
357
+ hooks = deep_merge_dicts(*hooks)
358
+
359
+ # Enable sync or not
277
360
  if sync or show:
278
361
  sync = True
279
362
  else:
280
363
  from secator.celery import is_celery_worker_alive
281
364
  worker_alive = is_celery_worker_alive()
282
- if not worker_alive:
365
+ if not worker_alive and not worker:
283
366
  sync = True
284
367
  else:
285
368
  sync = False
@@ -289,34 +372,28 @@ def register_runner(cli_endpoint, config):
289
372
  if (broker_protocol == 'redis' or backend_protocol == 'redis') and not ADDONS_ENABLED['redis']:
290
373
  _get_rich_console().print('[bold red]Missing `redis` addon: please run `secator install addons redis`[/].')
291
374
  sys.exit(1)
292
- opts['sync'] = sync
375
+
376
+ from secator.utils import debug
377
+ debug('Run options', obj=opts, sub='cli')
378
+
379
+ # Set run options
293
380
  opts.update({
294
- 'print_item': not sync,
295
- 'print_line': sync,
296
- 'print_remote_status': not sync,
297
- 'print_start': not sync
381
+ 'print_cmd': True,
382
+ 'print_item': True,
383
+ 'print_line': True,
384
+ 'print_progress': True,
385
+ 'print_remote_info': not sync,
386
+ 'piped_input': ctx.obj['piped_input'],
387
+ 'piped_output': ctx.obj['piped_output'],
388
+ 'caller': 'cli',
389
+ 'sync': sync,
298
390
  })
299
391
 
300
- # Build hooks from driver name
301
- hooks = {}
302
- if driver == 'mongodb':
303
- if not ADDONS_ENABLED['mongodb']:
304
- _get_rich_console().print('[bold red]Missing `mongodb` addon: please run `secator install addons mongodb`[/].')
305
- sys.exit(1)
306
- from secator.hooks.mongodb import MONGODB_HOOKS
307
- hooks = MONGODB_HOOKS
308
-
309
- # Build exporters
310
- runner = runner_cls(config, targets, run_opts=opts, hooks=hooks, context=context)
392
+ # Start runner
393
+ runner = runner_cls(config, inputs, run_opts=opts, hooks=hooks, context=context)
311
394
  runner.run()
312
395
 
313
- settings = {'ignore_unknown_options': False, 'allow_extra_args': False}
314
- cli_endpoint.command(
315
- name=config.name,
316
- context_settings=settings,
317
- no_args_is_help=no_args_is_help,
318
- short_help=short_help)(func)
319
-
396
+ generate_cli_subcommand(cli_endpoint, func, **command_opts)
320
397
  generate_rich_click_opt_groups(cli_endpoint, name, input_type, options)
321
398
 
322
399
 
secator/definitions.py CHANGED
@@ -24,14 +24,19 @@ DEBUG = CONFIG.debug.level
24
24
  DEBUG_COMPONENT = CONFIG.debug.component.split(',')
25
25
 
26
26
  # Default tasks settings
27
- DEFAULT_HTTPX_FLAGS = os.environ.get('DEFAULT_HTTPX_FLAGS', '-td')
28
- DEFAULT_KATANA_FLAGS = os.environ.get('DEFAULT_KATANA_FLAGS', '-jc -js-crawl -known-files all -or -ob')
29
27
  DEFAULT_NUCLEI_FLAGS = os.environ.get('DEFAULT_NUCLEI_FLAGS', '-stats -sj -si 20 -hm -or')
30
28
  DEFAULT_FEROXBUSTER_FLAGS = os.environ.get('DEFAULT_FEROXBUSTER_FLAGS', '--auto-bail --no-state')
31
29
 
32
30
  # Constants
33
31
  OPT_NOT_SUPPORTED = -1
34
32
  OPT_PIPE_INPUT = -1
33
+ STATE_COLORS = {
34
+ 'PENDING': 'dim yellow3',
35
+ 'RUNNING': 'bold yellow3',
36
+ 'SUCCESS': 'bold green',
37
+ 'FAILURE': 'bold red',
38
+ 'REVOKED': 'bold magenta'
39
+ }
35
40
 
36
41
  # Vocab
37
42
  ALIVE = 'alive'
@@ -54,6 +59,7 @@ FILTER_SIZE = 'filter_size'
54
59
  HEADER = 'header'
55
60
  HOST = 'host'
56
61
  IP = 'ip'
62
+ PROTOCOL = 'protocol'
57
63
  LINES = 'lines'
58
64
  METHOD = 'method'
59
65
  MATCH_CODES = 'match_codes'
@@ -119,7 +125,8 @@ ADDONS_ENABLED = {}
119
125
 
120
126
  for addon, module in [
121
127
  ('worker', 'eventlet'),
122
- ('google', 'gspread'),
128
+ ('gdrive', 'gspread'),
129
+ ('gcs', 'google.cloud.storage'),
123
130
  ('mongodb', 'pymongo'),
124
131
  ('redis', 'redis'),
125
132
  ('dev', 'flake8'),
@@ -1,10 +1,12 @@
1
1
  __all__ = [
2
- 'CsvExporter',
3
- 'GdriveExporter',
4
- 'JsonExporter',
5
- 'TableExporter',
6
- 'TxtExporter'
2
+ 'ConsoleExporter',
3
+ 'CsvExporter',
4
+ 'GdriveExporter',
5
+ 'JsonExporter',
6
+ 'TableExporter',
7
+ 'TxtExporter'
7
8
  ]
9
+ from secator.exporters.console import ConsoleExporter
8
10
  from secator.exporters.csv import CsvExporter
9
11
  from secator.exporters.gdrive import GdriveExporter
10
12
  from secator.exporters.json import JsonExporter
@@ -0,0 +1,10 @@
1
+ from secator.exporters._base import Exporter
2
+ from secator.rich import console
3
+
4
+
5
+ class ConsoleExporter(Exporter):
6
+ def send(self):
7
+ results = self.report.data['results']
8
+ for items in results.values():
9
+ for item in items:
10
+ console.print(item)