secator 0.16.4__tar.gz → 0.17.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of secator might be problematic. Click here for more details.
- {secator-0.16.4 → secator-0.17.0}/CHANGELOG.md +21 -0
- {secator-0.16.4 → secator-0.17.0}/PKG-INFO +1 -1
- {secator-0.16.4 → secator-0.17.0}/cloudbuild.yaml +4 -2
- {secator-0.16.4 → secator-0.17.0}/pyproject.toml +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/celery.py +83 -12
- {secator-0.16.4 → secator-0.17.0}/secator/celery_signals.py +2 -1
- {secator-0.16.4 → secator-0.17.0}/secator/cli.py +7 -1
- {secator-0.16.4 → secator-0.17.0}/secator/config.py +1 -0
- {secator-0.16.4 → secator-0.17.0}/secator/hooks/gcs.py +15 -3
- {secator-0.16.4 → secator-0.17.0}/secator/hooks/mongodb.py +75 -65
- {secator-0.16.4 → secator-0.17.0}/secator/installer.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/certificate.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/exploit.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/ip.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/progress.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/record.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/stat.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/state.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/subdomain.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/tag.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/target.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/user_account.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/vulnerability.py +1 -1
- {secator-0.16.4 → secator-0.17.0}/secator/runners/command.py +21 -6
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/_categories.py +14 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/cariddi.py +37 -1
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/dalfox.py +2 -2
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/dirsearch.py +0 -1
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/feroxbuster.py +0 -1
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/ffuf.py +0 -1
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/katana.py +3 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/naabu.py +1 -2
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/nuclei.py +4 -1
- {secator-0.16.4 → secator-0.17.0}/secator/utils.py +9 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/outputs.py +2 -2
- {secator-0.16.4 → secator-0.17.0}/tests/integration/test_tasks_categories.py +10 -1
- {secator-0.16.4 → secator-0.17.0}/.coderabbit.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/.docker/Dockerfile.alpine +0 -0
- {secator-0.16.4 → secator-0.17.0}/.docker/Dockerfile.arch +0 -0
- {secator-0.16.4 → secator-0.17.0}/.docker/Dockerfile.debian +0 -0
- {secator-0.16.4 → secator-0.17.0}/.docker/Dockerfile.kali +0 -0
- {secator-0.16.4 → secator-0.17.0}/.docker/Dockerfile.osx +0 -0
- {secator-0.16.4 → secator-0.17.0}/.docker/Dockerfile.ubuntu +0 -0
- {secator-0.16.4 → secator-0.17.0}/.docker/build_all.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/.dockerignore +0 -0
- {secator-0.16.4 → secator-0.17.0}/.flake8 +0 -0
- {secator-0.16.4 → secator-0.17.0}/.gitignore +0 -0
- {secator-0.16.4 → secator-0.17.0}/CONTRIBUTING.md +0 -0
- {secator-0.16.4 → secator-0.17.0}/Dockerfile +0 -0
- {secator-0.16.4 → secator-0.17.0}/LICENSE +0 -0
- {secator-0.16.4 → secator-0.17.0}/README.md +0 -0
- {secator-0.16.4 → secator-0.17.0}/SECURITY.md +0 -0
- {secator-0.16.4 → secator-0.17.0}/helm/.helmignore +0 -0
- {secator-0.16.4 → secator-0.17.0}/helm/Chart.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/helm/templates/redis-service.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/helm/templates/redis.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/helm/templates/secator-manager.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/helm/templates/secator-worker.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/helm/values.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/download_cves.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/generate_tools_md_table.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/install.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/install_asciinema.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/install_go.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/install_ruby.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/msf/exploit_cve.rc +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/msf/ftp_anonymous.rc +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/msf/ftp_version.rc +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/msf/ftp_vsftpd_234_backdoor.rc +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/msf/redis.rc +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/stories/STORY.md +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/stories/aliases.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/stories/demo.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/stories/fmt.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/stories/input.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/stories/pipe.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/stories/short_demo.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/scripts/update_tools.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/.gitignore +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/celery_utils.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/cli_helper.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/click.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/aggressive.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/http_headless.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/http_record.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/insane.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/paranoid.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/polite.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/sneaky.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/profiles/tor.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/scans/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/scans/domain.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/scans/host.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/scans/network.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/scans/subdomain.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/scans/url.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/cidr_recon.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/code_scan.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/host_recon.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/subdomain_recon.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/url_bypass.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/url_crawl.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/url_dirsearch.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/url_fuzz.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/url_params_fuzz.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/url_vuln.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/user_hunt.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/configs/workflows/wordpress.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/cve.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/decorators.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/definitions.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/_base.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/console.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/csv.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/gdrive.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/json.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/table.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/exporters/txt.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/hooks/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/loader.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/_base.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/error.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/info.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/port.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/url.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/output_types/warning.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/report.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/rich.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/runners/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/runners/_base.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/runners/_helpers.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/runners/celery.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/runners/scan.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/runners/task.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/runners/workflow.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/scans/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/serializers/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/serializers/dataclass.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/serializers/json.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/serializers/regex.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/arjun.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/bbot.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/bup.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/dnsx.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/fping.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/gau.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/gf.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/gitleaks.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/gospider.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/grype.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/h8mail.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/httpx.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/maigret.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/mapcidr.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/msfconsole.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/nmap.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/searchsploit.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/subfinder.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/testssl.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/trivy.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/wafw00f.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/wpprobe.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tasks/wpscan.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/template.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/thread.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/tree.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/utils_test.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/secator/workflows/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/fixtures/h8mail_breach.txt +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/fixtures/ls.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/fixtures/msfconsole_input.rc +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/fixtures/nmap_output.xml +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/all.yaml +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/inputs.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/setup.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/teardown.sh +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/test_addons.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/test_celery.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/test_scans.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/test_tasks.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/test_worker.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/test_workflows.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/wordlist.txt +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/wordlist_dns.txt +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/wordpress_toolbox/Dockerfile +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/integration/wordpress_toolbox/Makefile +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/performance/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/performance/loadtester.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/performance/test_worker.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/template/test_templates.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/__init__.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_celery.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_cli.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_command.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_config.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_offline.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_runners.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_runners_helpers.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_scans.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_serializers.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_tasks.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_tasks_categories.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_template.py +0 -0
- {secator-0.16.4 → secator-0.17.0}/tests/unit/test_utils.py +0 -0
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.17.0](https://github.com/freelabz/secator/compare/v0.16.5...v0.17.0) (2025-09-06)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* enforceable memory limits ([#710](https://github.com/freelabz/secator/issues/710)) ([eaefa1f](https://github.com/freelabz/secator/commit/eaefa1f5497af4d6a6020e09d5f954ae4639f20c))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* prod optimizations ([#708](https://github.com/freelabz/secator/issues/708)) ([efb5d3c](https://github.com/freelabz/secator/commit/efb5d3ce637a5de6e23ed157fd85d2a39bef360d))
|
|
14
|
+
|
|
15
|
+
## [0.16.5](https://github.com/freelabz/secator/compare/v0.16.4...v0.16.5) (2025-06-25)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
### Bug Fixes
|
|
19
|
+
|
|
20
|
+
* **celery:** pass mongodb uuids when enabled ([#701](https://github.com/freelabz/secator/issues/701)) ([64b43e8](https://github.com/freelabz/secator/commit/64b43e88659c963a0c526829a2f72ee75348edef))
|
|
21
|
+
* **ci:** add apt update in ci ([261d1e8](https://github.com/freelabz/secator/commit/261d1e8bdbca06e85adf3df7a9489bff7ba445ab))
|
|
22
|
+
* prod optimizations (GCS ValueError, dynamic profile for fuzzers with big wordlists) ([#707](https://github.com/freelabz/secator/issues/707)) ([bcd6024](https://github.com/freelabz/secator/commit/bcd6024d91362ca141b71a49c4f80c759e1801ca))
|
|
23
|
+
|
|
3
24
|
## [0.16.4](https://github.com/freelabz/secator/compare/v0.16.3...v0.16.4) (2025-06-13)
|
|
4
25
|
|
|
5
26
|
|
|
@@ -3,11 +3,13 @@ steps:
|
|
|
3
3
|
entrypoint: 'bash'
|
|
4
4
|
args: ['-c', 'docker pull ${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION} || exit 0']
|
|
5
5
|
- name: 'gcr.io/cloud-builders/docker'
|
|
6
|
-
args: ['build', '-t', '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}', '--cache-from', '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}', '.']
|
|
6
|
+
args: ['build', '-t', '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}', '--build-arg', 'flavor=${_FLAVOR}', '--cache-from', '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}', '.']
|
|
7
|
+
|
|
7
8
|
substitutions:
|
|
8
9
|
_REPOSITORY: secator
|
|
9
10
|
_LOCATION: europe-west1
|
|
10
|
-
_VERSION:
|
|
11
|
+
_VERSION: dev
|
|
12
|
+
_FLAVOR: full
|
|
11
13
|
|
|
12
14
|
images:
|
|
13
15
|
- '${_LOCATION}-docker.pkg.dev/$PROJECT_ID/${_REPOSITORY}/secator:${_VERSION}'
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import gc
|
|
1
2
|
import json
|
|
2
3
|
import logging
|
|
3
4
|
import os
|
|
@@ -5,6 +6,7 @@ import os
|
|
|
5
6
|
from time import time
|
|
6
7
|
|
|
7
8
|
from celery import Celery, chord
|
|
9
|
+
from celery.canvas import signature
|
|
8
10
|
from celery.app import trace
|
|
9
11
|
|
|
10
12
|
from rich.logging import RichHandler
|
|
@@ -61,9 +63,10 @@ app.conf.update({
|
|
|
61
63
|
'result_backend': CONFIG.celery.result_backend,
|
|
62
64
|
'result_expires': CONFIG.celery.result_expires,
|
|
63
65
|
'result_backend_transport_options': json.loads(CONFIG.celery.result_backend_transport_options) if CONFIG.celery.result_backend_transport_options else {}, # noqa: E501
|
|
64
|
-
'result_extended':
|
|
66
|
+
'result_extended': not CONFIG.addons.mongodb.enabled,
|
|
65
67
|
'result_backend_thread_safe': True,
|
|
66
68
|
'result_serializer': 'pickle',
|
|
69
|
+
'result_accept_content': ['application/x-python-serialize'],
|
|
67
70
|
|
|
68
71
|
# Task config
|
|
69
72
|
'task_acks_late': CONFIG.celery.task_acks_late,
|
|
@@ -81,6 +84,11 @@ app.conf.update({
|
|
|
81
84
|
'task_store_eager_result': True,
|
|
82
85
|
'task_send_sent_event': CONFIG.celery.task_send_sent_event,
|
|
83
86
|
'task_serializer': 'pickle',
|
|
87
|
+
'task_accept_content': ['application/x-python-serialize'],
|
|
88
|
+
|
|
89
|
+
# Event config
|
|
90
|
+
'event_serializer': 'pickle',
|
|
91
|
+
'event_accept_content': ['application/x-python-serialize'],
|
|
84
92
|
|
|
85
93
|
# Worker config
|
|
86
94
|
# 'worker_direct': True, # TODO: consider enabling this to allow routing to specific workers
|
|
@@ -168,6 +176,12 @@ def run_scan(self, args=[], kwargs={}):
|
|
|
168
176
|
|
|
169
177
|
@app.task(bind=True)
|
|
170
178
|
def run_command(self, results, name, targets, opts={}):
|
|
179
|
+
# Set Celery request id in context
|
|
180
|
+
context = opts.get('context', {})
|
|
181
|
+
context['celery_id'] = self.request.id
|
|
182
|
+
context['worker_name'] = os.environ.get('WORKER_NAME', 'unknown')
|
|
183
|
+
|
|
184
|
+
# Set routing key in context
|
|
171
185
|
if IN_CELERY_WORKER_PROCESS:
|
|
172
186
|
quiet = not CONFIG.cli.worker_command_verbose
|
|
173
187
|
opts.update({
|
|
@@ -179,15 +193,13 @@ def run_command(self, results, name, targets, opts={}):
|
|
|
179
193
|
'quiet': quiet
|
|
180
194
|
})
|
|
181
195
|
routing_key = self.request.delivery_info['routing_key']
|
|
196
|
+
context['routing_key'] = routing_key
|
|
182
197
|
debug(f'Task "{name}" running with routing key "{routing_key}"', sub='celery.state')
|
|
183
198
|
|
|
184
199
|
# Flatten + dedupe + filter results
|
|
185
200
|
results = forward_results(results)
|
|
186
201
|
|
|
187
|
-
# Set
|
|
188
|
-
context = opts.get('context', {})
|
|
189
|
-
context['celery_id'] = self.request.id
|
|
190
|
-
context['worker_name'] = os.environ.get('WORKER_NAME', 'unknown')
|
|
202
|
+
# Set task opts
|
|
191
203
|
opts['context'] = context
|
|
192
204
|
opts['results'] = results
|
|
193
205
|
opts['sync'] = True
|
|
@@ -204,10 +216,13 @@ def run_command(self, results, name, targets, opts={}):
|
|
|
204
216
|
# Chunk task if needed
|
|
205
217
|
if chunk_it:
|
|
206
218
|
if IN_CELERY_WORKER_PROCESS:
|
|
207
|
-
console.print(Info(message=f'Task {name} requires chunking
|
|
208
|
-
|
|
219
|
+
console.print(Info(message=f'Task {name} requires chunking'))
|
|
220
|
+
workflow = break_task(task, opts, results=results)
|
|
221
|
+
if IN_CELERY_WORKER_PROCESS:
|
|
222
|
+
console.print(Info(message=f'Task {name} successfully broken into {len(workflow)} chunks'))
|
|
209
223
|
update_state(self, task, force=True)
|
|
210
|
-
|
|
224
|
+
console.print(Info(message=f'Task {name} updated state, replacing task with Celery chord workflow'))
|
|
225
|
+
return replace(self, workflow)
|
|
211
226
|
|
|
212
227
|
# Update state live
|
|
213
228
|
for _ in task:
|
|
@@ -240,7 +255,8 @@ def forward_results(results):
|
|
|
240
255
|
console.print(Info(message=f'Deduplicating {len(results)} results'))
|
|
241
256
|
|
|
242
257
|
results = flatten(results)
|
|
243
|
-
if CONFIG.addons.mongodb.enabled:
|
|
258
|
+
if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
|
|
259
|
+
console.print(Info(message=f'Extracting uuids from {len(results)} results'))
|
|
244
260
|
uuids = [r._uuid for r in results if hasattr(r, '_uuid')]
|
|
245
261
|
uuids.extend([r for r in results if isinstance(r, str)])
|
|
246
262
|
results = list(set(uuids))
|
|
@@ -271,12 +287,14 @@ def mark_runner_started(results, runner, enable_hooks=True):
|
|
|
271
287
|
if results:
|
|
272
288
|
results = forward_results(results)
|
|
273
289
|
runner.enable_hooks = enable_hooks
|
|
274
|
-
if CONFIG.addons.mongodb.enabled:
|
|
290
|
+
if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
|
|
275
291
|
from secator.hooks.mongodb import get_results
|
|
276
292
|
results = get_results(results)
|
|
277
293
|
for item in results:
|
|
278
294
|
runner.add_result(item, print=False)
|
|
279
295
|
runner.mark_started()
|
|
296
|
+
if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
|
|
297
|
+
return [r._uuid for r in runner.results]
|
|
280
298
|
return runner.results
|
|
281
299
|
|
|
282
300
|
|
|
@@ -297,12 +315,14 @@ def mark_runner_completed(results, runner, enable_hooks=True):
|
|
|
297
315
|
debug(f'Runner {runner.unique_name} has finished, running mark_completed', sub='celery')
|
|
298
316
|
results = forward_results(results)
|
|
299
317
|
runner.enable_hooks = enable_hooks
|
|
300
|
-
if CONFIG.addons.mongodb.enabled:
|
|
318
|
+
if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
|
|
301
319
|
from secator.hooks.mongodb import get_results
|
|
302
320
|
results = get_results(results)
|
|
303
321
|
for item in results:
|
|
304
322
|
runner.add_result(item, print=False)
|
|
305
323
|
runner.mark_completed()
|
|
324
|
+
if IN_CELERY_WORKER_PROCESS and CONFIG.addons.mongodb.enabled:
|
|
325
|
+
return [r._uuid for r in runner.results]
|
|
306
326
|
return runner.results
|
|
307
327
|
|
|
308
328
|
|
|
@@ -322,6 +342,52 @@ def is_celery_worker_alive():
|
|
|
322
342
|
return result
|
|
323
343
|
|
|
324
344
|
|
|
345
|
+
def replace(task_instance, sig):
|
|
346
|
+
"""Replace this task, with a new task inheriting the task id.
|
|
347
|
+
|
|
348
|
+
Execution of the host task ends immediately and no subsequent statements
|
|
349
|
+
will be run.
|
|
350
|
+
|
|
351
|
+
.. versionadded:: 4.0
|
|
352
|
+
|
|
353
|
+
Arguments:
|
|
354
|
+
sig (Signature): signature to replace with.
|
|
355
|
+
visitor (StampingVisitor): Visitor API object.
|
|
356
|
+
|
|
357
|
+
Raises:
|
|
358
|
+
~@Ignore: This is always raised when called in asynchronous context.
|
|
359
|
+
It is best to always use ``return self.replace(...)`` to convey
|
|
360
|
+
to the reader that the task won't continue after being replaced.
|
|
361
|
+
"""
|
|
362
|
+
console.print('Replacing task')
|
|
363
|
+
chord = task_instance.request.chord
|
|
364
|
+
sig.freeze(task_instance.request.id)
|
|
365
|
+
replaced_task_nesting = task_instance.request.get('replaced_task_nesting', 0) + 1
|
|
366
|
+
sig.set(
|
|
367
|
+
chord=chord,
|
|
368
|
+
group_id=task_instance.request.group,
|
|
369
|
+
group_index=task_instance.request.group_index,
|
|
370
|
+
root_id=task_instance.request.root_id,
|
|
371
|
+
replaced_task_nesting=replaced_task_nesting
|
|
372
|
+
)
|
|
373
|
+
import psutil
|
|
374
|
+
import os
|
|
375
|
+
process = psutil.Process(os.getpid())
|
|
376
|
+
length = len(task_instance.request.chain) if task_instance.request.chain else 0
|
|
377
|
+
console.print(f'Adding {length} chain tasks from request chain')
|
|
378
|
+
for ix, t in enumerate(reversed(task_instance.request.chain or [])):
|
|
379
|
+
console.print(f'Adding chain task {t.name} from request chain ({ix + 1}/{length})')
|
|
380
|
+
chain_task = signature(t, app=task_instance.app)
|
|
381
|
+
chain_task.set(replaced_task_nesting=replaced_task_nesting)
|
|
382
|
+
sig |= chain_task
|
|
383
|
+
del chain_task
|
|
384
|
+
del t
|
|
385
|
+
memory_bytes = process.memory_info().rss
|
|
386
|
+
console.print(f'Memory usage: {memory_bytes / 1024 / 1024:.2f} MB (chain task {ix + 1}/{length})')
|
|
387
|
+
gc.collect()
|
|
388
|
+
return task_instance.on_replace(sig)
|
|
389
|
+
|
|
390
|
+
|
|
325
391
|
def break_task(task, task_opts, results=[]):
|
|
326
392
|
"""Break a task into multiple of the same type."""
|
|
327
393
|
chunks = task.inputs
|
|
@@ -365,16 +431,21 @@ def break_task(task, task_opts, results=[]):
|
|
|
365
431
|
task_id = sig.freeze().task_id
|
|
366
432
|
full_name = f'{task.name}_{ix + 1}'
|
|
367
433
|
task.add_subtask(task_id, task.name, full_name)
|
|
368
|
-
info = Info(message=f'Celery chunked task created: {task_id}')
|
|
434
|
+
info = Info(message=f'Celery chunked task created ({ix + 1} / {len(chunks)}): {task_id}')
|
|
369
435
|
task.add_result(info)
|
|
370
436
|
sigs.append(sig)
|
|
371
437
|
|
|
372
438
|
# Mark main task as async since it's being chunked
|
|
373
439
|
task.sync = False
|
|
440
|
+
task.results = []
|
|
441
|
+
task.uuids = set()
|
|
442
|
+
console.print(Info(message=f'Task {task.unique_name} is now async, building chord with{len(sigs)} chunks'))
|
|
443
|
+
console.print(Info(message=f'Results: {results}'))
|
|
374
444
|
|
|
375
445
|
# Build Celery workflow
|
|
376
446
|
workflow = chord(
|
|
377
447
|
tuple(sigs),
|
|
378
448
|
mark_runner_completed.s(runner=task).set(queue='results')
|
|
379
449
|
)
|
|
450
|
+
console.print(Info(message=f'Task {task.unique_name} chord built with {len(sigs)} chunks, returning workflow'))
|
|
380
451
|
return workflow
|
|
@@ -92,8 +92,9 @@ def task_postrun_handler(**kwargs):
|
|
|
92
92
|
|
|
93
93
|
# Get sender name from kwargs
|
|
94
94
|
sender_name = kwargs['sender'].name
|
|
95
|
+
# console.print(Info(message=f'Task postrun handler --> Sender name: {sender_name}'))
|
|
95
96
|
|
|
96
|
-
if CONFIG.celery.worker_kill_after_task and sender_name.startswith('secator.'):
|
|
97
|
+
if CONFIG.celery.worker_kill_after_task and (sender_name.startswith('secator.') or sender_name.startswith('api.')):
|
|
97
98
|
worker_name = os.environ.get('WORKER_NAME', 'unknown')
|
|
98
99
|
console.print(Info(message=f'Shutdown worker {worker_name} since config celery.worker_kill_after_task is set.'))
|
|
99
100
|
kill_worker(parent=True)
|
|
@@ -134,7 +134,10 @@ for config in SCANS:
|
|
|
134
134
|
@click.option('--stop', is_flag=True, help='Stop a worker in dev mode (celery multi).')
|
|
135
135
|
@click.option('--show', is_flag=True, help='Show command (celery multi).')
|
|
136
136
|
@click.option('--use-command-runner', is_flag=True, default=False, help='Use command runner to run the command.')
|
|
137
|
-
|
|
137
|
+
@click.option('--without-gossip', is_flag=True)
|
|
138
|
+
@click.option('--without-mingle', is_flag=True)
|
|
139
|
+
@click.option('--without-heartbeat', is_flag=True)
|
|
140
|
+
def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, dev, stop, show, use_command_runner, without_gossip, without_mingle, without_heartbeat): # noqa: E501
|
|
138
141
|
"""Run a worker."""
|
|
139
142
|
|
|
140
143
|
# Check Celery addon is installed
|
|
@@ -182,6 +185,9 @@ def worker(hostname, concurrency, reload, queue, pool, quiet, loglevel, check, d
|
|
|
182
185
|
cmd += f' -P {pool}' if pool else ''
|
|
183
186
|
cmd += f' -c {concurrency}' if concurrency else ''
|
|
184
187
|
cmd += f' -l {loglevel}' if loglevel else ''
|
|
188
|
+
cmd += ' --without-mingle' if without_mingle else ''
|
|
189
|
+
cmd += ' --without-gossip' if without_gossip else ''
|
|
190
|
+
cmd += ' --without-heartbeat' if without_heartbeat else ''
|
|
185
191
|
|
|
186
192
|
if reload:
|
|
187
193
|
patterns = "celery.py;tasks/*.py;runners/*.py;serializers/*.py;output_types/*.py;hooks/*.py;exporters/*.py"
|
|
@@ -14,6 +14,16 @@ ITEMS_TO_SEND = {
|
|
|
14
14
|
'url': ['screenshot_path', 'stored_response_path']
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
+
_gcs_client = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_gcs_client():
|
|
21
|
+
"""Get or create GCS client"""
|
|
22
|
+
global _gcs_client
|
|
23
|
+
if _gcs_client is None:
|
|
24
|
+
_gcs_client = storage.Client()
|
|
25
|
+
return _gcs_client
|
|
26
|
+
|
|
17
27
|
|
|
18
28
|
def process_item(self, item):
|
|
19
29
|
if item._type not in ITEMS_TO_SEND.keys():
|
|
@@ -39,13 +49,15 @@ def process_item(self, item):
|
|
|
39
49
|
def upload_blob(bucket_name, source_file_name, destination_blob_name):
|
|
40
50
|
"""Uploads a file to the bucket."""
|
|
41
51
|
start_time = time()
|
|
42
|
-
storage_client =
|
|
52
|
+
storage_client = get_gcs_client()
|
|
43
53
|
bucket = storage_client.bucket(bucket_name)
|
|
44
54
|
blob = bucket.blob(destination_blob_name)
|
|
45
|
-
|
|
55
|
+
with open(source_file_name, 'rb') as f:
|
|
56
|
+
f.seek(0)
|
|
57
|
+
blob.upload_from_file(f)
|
|
46
58
|
end_time = time()
|
|
47
59
|
elapsed = end_time - start_time
|
|
48
|
-
debug(f'in {elapsed:.4f}s', obj={'blob': '
|
|
60
|
+
debug(f'in {elapsed:.4f}s', obj={'blob': 'UPLOADED', 'blob_name': destination_blob_name, 'bucket': bucket_name}, obj_after=False, sub='hooks.gcs', verbose=True) # noqa: E501
|
|
49
61
|
|
|
50
62
|
|
|
51
63
|
HOOKS = {
|
|
@@ -30,7 +30,8 @@ def get_mongodb_client():
|
|
|
30
30
|
_mongodb_client = pymongo.MongoClient(
|
|
31
31
|
escape_mongodb_url(MONGODB_URL),
|
|
32
32
|
maxPoolSize=MONGODB_MAX_POOL_SIZE,
|
|
33
|
-
serverSelectionTimeoutMS=MONGODB_CONNECT_TIMEOUT
|
|
33
|
+
serverSelectionTimeoutMS=MONGODB_CONNECT_TIMEOUT,
|
|
34
|
+
connect=False
|
|
34
35
|
)
|
|
35
36
|
return _mongodb_client
|
|
36
37
|
|
|
@@ -157,89 +158,98 @@ def load_findings(objs):
|
|
|
157
158
|
|
|
158
159
|
|
|
159
160
|
@shared_task
|
|
160
|
-
def tag_duplicates(ws_id: str = None):
|
|
161
|
+
def tag_duplicates(ws_id: str = None, full_scan: bool = False):
|
|
161
162
|
"""Tag duplicates in workspace.
|
|
162
163
|
|
|
163
164
|
Args:
|
|
164
165
|
ws_id (str): Workspace id.
|
|
166
|
+
full_scan (bool): If True, scan all findings, otherwise only untagged findings.
|
|
165
167
|
"""
|
|
166
168
|
debug(f'running duplicate check on workspace {ws_id}', sub='hooks.mongodb')
|
|
169
|
+
init_time = time.time()
|
|
167
170
|
client = get_mongodb_client()
|
|
168
171
|
db = client.main
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
untagged_query =
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
debug(
|
|
172
|
+
start_time = time.time()
|
|
173
|
+
workspace_query = {'_context.workspace_id': str(ws_id), '_context.workspace_duplicate': False, '_tagged': True}
|
|
174
|
+
untagged_query = {'_context.workspace_id': str(ws_id), '_tagged': {'$ne': True}}
|
|
175
|
+
if full_scan:
|
|
176
|
+
del untagged_query['_tagged']
|
|
177
|
+
workspace_findings = load_findings(list(db.findings.find(workspace_query).sort('_timestamp', -1)))
|
|
178
|
+
untagged_findings = load_findings(list(db.findings.find(untagged_query).sort('_timestamp', -1)))
|
|
179
|
+
debug(
|
|
180
|
+
f'Workspace non-duplicates findings: {len(workspace_findings)}, '
|
|
181
|
+
f'Untagged findings: {len(untagged_findings)}. '
|
|
182
|
+
f'Query time: {time.time() - start_time}s',
|
|
183
|
+
sub='hooks.mongodb'
|
|
184
|
+
)
|
|
185
|
+
start_time = time.time()
|
|
186
|
+
seen = []
|
|
187
|
+
db_updates = {}
|
|
177
188
|
|
|
178
|
-
untagged_findings = load_findings(untagged_query)
|
|
179
|
-
workspace_findings = load_findings(workspace_query)
|
|
180
|
-
non_duplicates = []
|
|
181
|
-
duplicates = []
|
|
182
189
|
for item in untagged_findings:
|
|
183
|
-
|
|
184
|
-
seen = [f for f in duplicates if f._uuid == item._uuid]
|
|
185
|
-
if seen:
|
|
190
|
+
if item._uuid in seen:
|
|
186
191
|
continue
|
|
187
192
|
|
|
188
|
-
|
|
189
|
-
|
|
193
|
+
debug(
|
|
194
|
+
f'Processing: {repr(item)} ({item._timestamp}) [{item._uuid}]',
|
|
195
|
+
sub='hooks.mongodb',
|
|
196
|
+
verbose=True
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
duplicate_ids = [
|
|
200
|
+
_._uuid
|
|
201
|
+
for _ in untagged_findings
|
|
202
|
+
if _ == item and _._uuid != item._uuid
|
|
203
|
+
]
|
|
204
|
+
seen.extend(duplicate_ids)
|
|
190
205
|
|
|
191
|
-
# Check if already present in list of workspace_findings findings, list of duplicates, or untagged_findings
|
|
192
|
-
workspace_dupes = [f for f in workspace_findings if f == item and f._uuid != item._uuid]
|
|
193
|
-
untagged_dupes = [f for f in untagged_findings if f == item and f._uuid != item._uuid]
|
|
194
|
-
seen_dupes = [f for f in duplicates if f == item and f._uuid != item._uuid]
|
|
195
|
-
tmp_duplicates.extend(workspace_dupes)
|
|
196
|
-
tmp_duplicates.extend(untagged_dupes)
|
|
197
|
-
tmp_duplicates.extend(seen_dupes)
|
|
198
206
|
debug(
|
|
199
|
-
f'for item
|
|
200
|
-
obj={
|
|
201
|
-
'workspace dupes': len(workspace_dupes),
|
|
202
|
-
'untagged dupes': len(untagged_dupes),
|
|
203
|
-
'seen dupes': len(seen_dupes)
|
|
204
|
-
},
|
|
205
|
-
id=ws_id,
|
|
207
|
+
f'Found {len(duplicate_ids)} duplicates for item',
|
|
206
208
|
sub='hooks.mongodb',
|
|
207
|
-
verbose=True
|
|
208
|
-
|
|
209
|
-
debug(f'duplicate ids: {tmp_duplicates_ids}', id=ws_id, sub='hooks.mongodb', verbose=True)
|
|
210
|
-
|
|
211
|
-
# Update latest object as non-duplicate
|
|
212
|
-
if tmp_duplicates:
|
|
213
|
-
duplicates.extend([f for f in tmp_duplicates])
|
|
214
|
-
db.findings.update_one({'_id': ObjectId(item._uuid)}, {'$set': {'_related': tmp_duplicates_ids}})
|
|
215
|
-
debug(f'adding {item._uuid} as non-duplicate', id=ws_id, sub='hooks.mongodb', verbose=True)
|
|
216
|
-
non_duplicates.append(item)
|
|
217
|
-
else:
|
|
218
|
-
debug(f'adding {item._uuid} as non-duplicate', id=ws_id, sub='hooks.mongodb', verbose=True)
|
|
219
|
-
non_duplicates.append(item)
|
|
209
|
+
verbose=True
|
|
210
|
+
)
|
|
220
211
|
|
|
221
|
-
|
|
212
|
+
duplicate_ws = [
|
|
213
|
+
_ for _ in workspace_findings
|
|
214
|
+
if _ == item and _._uuid != item._uuid
|
|
215
|
+
]
|
|
216
|
+
debug(f' --> Found {len(duplicate_ws)} workspace duplicates for item', sub='hooks.mongodb', verbose=True)
|
|
217
|
+
|
|
218
|
+
related_ids = []
|
|
219
|
+
if duplicate_ws:
|
|
220
|
+
duplicate_ws_ids = [_._uuid for _ in duplicate_ws]
|
|
221
|
+
duplicate_ids.extend(duplicate_ws_ids)
|
|
222
|
+
for related in duplicate_ws:
|
|
223
|
+
related_ids.extend(related._related)
|
|
224
|
+
|
|
225
|
+
debug(f' --> Found {len(duplicate_ids)} total duplicates for item', sub='hooks.mongodb', verbose=True)
|
|
226
|
+
|
|
227
|
+
db_updates[item._uuid] = {
|
|
228
|
+
'_related': duplicate_ids + related_ids,
|
|
229
|
+
'_context.workspace_duplicate': False,
|
|
230
|
+
'_tagged': True
|
|
231
|
+
}
|
|
232
|
+
for uuid in duplicate_ids:
|
|
233
|
+
db_updates[uuid] = {
|
|
234
|
+
'_context.workspace_duplicate': True,
|
|
235
|
+
'_tagged': True
|
|
236
|
+
}
|
|
237
|
+
debug(f'Finished processing untagged findings in {time.time() - start_time}s', sub='hooks.mongodb')
|
|
238
|
+
start_time = time.time()
|
|
222
239
|
|
|
223
|
-
|
|
224
|
-
duplicates_ids = list(dict.fromkeys([n._uuid for n in duplicates]))
|
|
225
|
-
non_duplicates_ids = list(dict.fromkeys([n._uuid for n in non_duplicates]))
|
|
240
|
+
debug(f'Executing {len(db_updates)} database updates', sub='hooks.mongodb')
|
|
226
241
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
242
|
+
from pymongo import UpdateOne
|
|
243
|
+
if not db_updates:
|
|
244
|
+
debug('no db updates to execute', sub='hooks.mongodb')
|
|
245
|
+
return
|
|
230
246
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
debug(
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
obj={
|
|
238
|
-
'processed': len(untagged_findings),
|
|
239
|
-
'duplicates': len(duplicates_ids),
|
|
240
|
-
'non-duplicates': len(non_duplicates_ids)
|
|
241
|
-
},
|
|
242
|
-
sub='hooks.mongodb')
|
|
247
|
+
result = db.findings.bulk_write(
|
|
248
|
+
[UpdateOne({'_id': ObjectId(uuid)}, {'$set': update}) for uuid, update in db_updates.items()]
|
|
249
|
+
)
|
|
250
|
+
debug(result, sub='hooks.mongodb')
|
|
251
|
+
debug(f'Finished running db update in {time.time() - start_time}s', sub='hooks.mongodb')
|
|
252
|
+
debug(f'Finished running tag duplicates in {time.time() - init_time}s', sub='hooks.mongodb')
|
|
243
253
|
|
|
244
254
|
|
|
245
255
|
HOOKS = {
|
|
@@ -395,7 +395,7 @@ class GithubInstaller:
|
|
|
395
395
|
for root, _, files in os.walk(directory):
|
|
396
396
|
for file in files:
|
|
397
397
|
# Match the file name exactly with the repository name
|
|
398
|
-
if file
|
|
398
|
+
if file.startswith(binary_name):
|
|
399
399
|
return os.path.join(root, file)
|
|
400
400
|
return None
|
|
401
401
|
|
|
@@ -26,7 +26,7 @@ class Certificate(OutputType):
|
|
|
26
26
|
serial_number: str = field(default='', compare=False)
|
|
27
27
|
ciphers: list[str] = field(default_factory=list, compare=False)
|
|
28
28
|
# parent_certificate: 'Certificate' = None # noqa: F821
|
|
29
|
-
_source: str = field(default='', repr=True)
|
|
29
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
30
30
|
_type: str = field(default='certificate', repr=True)
|
|
31
31
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
32
32
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -18,7 +18,7 @@ class Exploit(OutputType):
|
|
|
18
18
|
cves: list = field(default_factory=list, compare=False)
|
|
19
19
|
tags: list = field(default_factory=list, compare=False)
|
|
20
20
|
extra_data: dict = field(default_factory=dict, compare=False)
|
|
21
|
-
_source: str = field(default='', repr=True)
|
|
21
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
22
22
|
_type: str = field(default='exploit', repr=True)
|
|
23
23
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
24
24
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -18,7 +18,7 @@ class Ip(OutputType):
|
|
|
18
18
|
host: str = field(default='', repr=True, compare=False)
|
|
19
19
|
alive: bool = False
|
|
20
20
|
protocol: str = field(default=IpProtocol.IPv4)
|
|
21
|
-
_source: str = field(default='', repr=True)
|
|
21
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
22
22
|
_type: str = field(default='ip', repr=True)
|
|
23
23
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
24
24
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -9,7 +9,7 @@ from secator.utils import rich_to_ansi, format_object
|
|
|
9
9
|
class Progress(OutputType):
|
|
10
10
|
percent: int = 0
|
|
11
11
|
extra_data: dict = field(default_factory=dict)
|
|
12
|
-
_source: str = field(default='', repr=True)
|
|
12
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
13
13
|
_type: str = field(default='progress', repr=True)
|
|
14
14
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
15
15
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -12,7 +12,7 @@ class Record(OutputType):
|
|
|
12
12
|
type: str
|
|
13
13
|
host: str = ''
|
|
14
14
|
extra_data: dict = field(default_factory=dict, compare=False)
|
|
15
|
-
_source: str = field(default='', repr=True)
|
|
15
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
16
16
|
_type: str = field(default='record', repr=True)
|
|
17
17
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
18
18
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -13,7 +13,7 @@ class Stat(OutputType):
|
|
|
13
13
|
memory: int
|
|
14
14
|
net_conns: int = field(default=None, repr=True)
|
|
15
15
|
extra_data: dict = field(default_factory=dict)
|
|
16
|
-
_source: str = field(default='', repr=True)
|
|
16
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
17
17
|
_type: str = field(default='stat', repr=True)
|
|
18
18
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
19
19
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -12,7 +12,7 @@ class State(OutputType):
|
|
|
12
12
|
task_id: str
|
|
13
13
|
state: str
|
|
14
14
|
_type: str = field(default='state', repr=True)
|
|
15
|
-
_source: str = field(default='', repr=True)
|
|
15
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
16
16
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
17
17
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
18
18
|
_context: dict = field(default_factory=dict, repr=True, compare=False)
|
|
@@ -13,7 +13,7 @@ class Subdomain(OutputType):
|
|
|
13
13
|
domain: str
|
|
14
14
|
sources: List[str] = field(default_factory=list, compare=False)
|
|
15
15
|
extra_data: dict = field(default_factory=dict, compare=False)
|
|
16
|
-
_source: str = field(default='', repr=True)
|
|
16
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
17
17
|
_type: str = field(default='subdomain', repr=True)
|
|
18
18
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
19
19
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -11,7 +11,7 @@ class Tag(OutputType):
|
|
|
11
11
|
match: str
|
|
12
12
|
extra_data: dict = field(default_factory=dict, repr=True, compare=False)
|
|
13
13
|
stored_response_path: str = field(default='', compare=False)
|
|
14
|
-
_source: str = field(default='', repr=True)
|
|
14
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
15
15
|
_type: str = field(default='tag', repr=True)
|
|
16
16
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
17
17
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -9,7 +9,7 @@ from secator.utils import autodetect_type, rich_to_ansi, rich_escape as _s
|
|
|
9
9
|
class Target(OutputType):
|
|
10
10
|
name: str
|
|
11
11
|
type: str = ''
|
|
12
|
-
_source: str = field(default='', repr=True)
|
|
12
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
13
13
|
_type: str = field(default='target', repr=True)
|
|
14
14
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
15
15
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -13,7 +13,7 @@ class UserAccount(OutputType):
|
|
|
13
13
|
email: str = ''
|
|
14
14
|
site_name: str = ''
|
|
15
15
|
extra_data: dict = field(default_factory=dict, compare=False)
|
|
16
|
-
_source: str = field(default='', repr=True)
|
|
16
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
17
17
|
_type: str = field(default='user_account', repr=True)
|
|
18
18
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
19
19
|
_uuid: str = field(default='', repr=True, compare=False)
|
|
@@ -25,7 +25,7 @@ class Vulnerability(OutputType):
|
|
|
25
25
|
reference: str = field(default='', compare=False)
|
|
26
26
|
confidence_nb: int = 0
|
|
27
27
|
severity_nb: int = 0
|
|
28
|
-
_source: str = field(default='', repr=True)
|
|
28
|
+
_source: str = field(default='', repr=True, compare=False)
|
|
29
29
|
_type: str = field(default='vulnerability', repr=True)
|
|
30
30
|
_timestamp: int = field(default_factory=lambda: time.time(), compare=False)
|
|
31
31
|
_uuid: str = field(default='', repr=True, compare=False)
|