clue-api 1.0.0.dev7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clue_api-1.0.0.dev7/LICENSE +11 -0
- clue_api-1.0.0.dev7/PKG-INFO +111 -0
- clue_api-1.0.0.dev7/README.md +56 -0
- clue_api-1.0.0.dev7/clue/.gitignore +21 -0
- clue_api-1.0.0.dev7/clue/__init__.py +0 -0
- clue_api-1.0.0.dev7/clue/api/__init__.py +211 -0
- clue_api-1.0.0.dev7/clue/api/base.py +99 -0
- clue_api-1.0.0.dev7/clue/api/v1/__init__.py +82 -0
- clue_api-1.0.0.dev7/clue/api/v1/actions.py +92 -0
- clue_api-1.0.0.dev7/clue/api/v1/auth.py +243 -0
- clue_api-1.0.0.dev7/clue/api/v1/configs.py +83 -0
- clue_api-1.0.0.dev7/clue/api/v1/fetchers.py +94 -0
- clue_api-1.0.0.dev7/clue/api/v1/lookup.py +221 -0
- clue_api-1.0.0.dev7/clue/api/v1/registration.py +109 -0
- clue_api-1.0.0.dev7/clue/api/v1/static.py +94 -0
- clue_api-1.0.0.dev7/clue/app.py +166 -0
- clue_api-1.0.0.dev7/clue/cache/__init__.py +129 -0
- clue_api-1.0.0.dev7/clue/common/__init__.py +0 -0
- clue_api-1.0.0.dev7/clue/common/classification.py +1006 -0
- clue_api-1.0.0.dev7/clue/common/classification.yml +130 -0
- clue_api-1.0.0.dev7/clue/common/dict_utils.py +130 -0
- clue_api-1.0.0.dev7/clue/common/exceptions.py +199 -0
- clue_api-1.0.0.dev7/clue/common/forge.py +152 -0
- clue_api-1.0.0.dev7/clue/common/json_utils.py +10 -0
- clue_api-1.0.0.dev7/clue/common/list_utils.py +11 -0
- clue_api-1.0.0.dev7/clue/common/logging/__init__.py +291 -0
- clue_api-1.0.0.dev7/clue/common/logging/audit.py +157 -0
- clue_api-1.0.0.dev7/clue/common/logging/format.py +42 -0
- clue_api-1.0.0.dev7/clue/common/regex.py +31 -0
- clue_api-1.0.0.dev7/clue/common/str_utils.py +213 -0
- clue_api-1.0.0.dev7/clue/common/swagger.py +139 -0
- clue_api-1.0.0.dev7/clue/common/uid.py +47 -0
- clue_api-1.0.0.dev7/clue/config.py +60 -0
- clue_api-1.0.0.dev7/clue/constants/__init__.py +0 -0
- clue_api-1.0.0.dev7/clue/constants/supported_types.py +38 -0
- clue_api-1.0.0.dev7/clue/cronjobs/__init__.py +30 -0
- clue_api-1.0.0.dev7/clue/cronjobs/plugins.py +32 -0
- clue_api-1.0.0.dev7/clue/error.py +129 -0
- clue_api-1.0.0.dev7/clue/gunicorn_config.py +29 -0
- clue_api-1.0.0.dev7/clue/healthz.py +74 -0
- clue_api-1.0.0.dev7/clue/helper/discover.py +53 -0
- clue_api-1.0.0.dev7/clue/helper/headers.py +30 -0
- clue_api-1.0.0.dev7/clue/helper/oauth.py +128 -0
- clue_api-1.0.0.dev7/clue/models/__init__.py +0 -0
- clue_api-1.0.0.dev7/clue/models/actions.py +243 -0
- clue_api-1.0.0.dev7/clue/models/config.py +456 -0
- clue_api-1.0.0.dev7/clue/models/fetchers.py +136 -0
- clue_api-1.0.0.dev7/clue/models/graph.py +162 -0
- clue_api-1.0.0.dev7/clue/models/model_list.py +52 -0
- clue_api-1.0.0.dev7/clue/models/network.py +430 -0
- clue_api-1.0.0.dev7/clue/models/results/__init__.py +34 -0
- clue_api-1.0.0.dev7/clue/models/results/base.py +10 -0
- clue_api-1.0.0.dev7/clue/models/results/graph.py +26 -0
- clue_api-1.0.0.dev7/clue/models/results/image.py +22 -0
- clue_api-1.0.0.dev7/clue/models/results/status.py +55 -0
- clue_api-1.0.0.dev7/clue/models/results/validation.py +57 -0
- clue_api-1.0.0.dev7/clue/models/selector.py +67 -0
- clue_api-1.0.0.dev7/clue/models/utils.py +52 -0
- clue_api-1.0.0.dev7/clue/models/validators.py +19 -0
- clue_api-1.0.0.dev7/clue/patched.py +8 -0
- clue_api-1.0.0.dev7/clue/plugin/__init__.py +1008 -0
- clue_api-1.0.0.dev7/clue/plugin/helpers/__init__.py +0 -0
- clue_api-1.0.0.dev7/clue/plugin/helpers/central_server.py +27 -0
- clue_api-1.0.0.dev7/clue/plugin/helpers/email_render.py +228 -0
- clue_api-1.0.0.dev7/clue/plugin/helpers/token.py +34 -0
- clue_api-1.0.0.dev7/clue/plugin/helpers/trino.py +103 -0
- clue_api-1.0.0.dev7/clue/plugin/interactive.py +270 -0
- clue_api-1.0.0.dev7/clue/plugin/models.py +19 -0
- clue_api-1.0.0.dev7/clue/plugin/utils.py +78 -0
- clue_api-1.0.0.dev7/clue/remote/__init__.py +0 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/__init__.py +130 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/cache.py +62 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/events.py +118 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/hash.py +193 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/queues/__init__.py +0 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/queues/comms.py +62 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/set.py +96 -0
- clue_api-1.0.0.dev7/clue/remote/datatypes/user_quota_tracker.py +54 -0
- clue_api-1.0.0.dev7/clue/security/__init__.py +211 -0
- clue_api-1.0.0.dev7/clue/security/obo.py +95 -0
- clue_api-1.0.0.dev7/clue/security/utils.py +34 -0
- clue_api-1.0.0.dev7/clue/services/action_service.py +186 -0
- clue_api-1.0.0.dev7/clue/services/auth_service.py +348 -0
- clue_api-1.0.0.dev7/clue/services/config_service.py +38 -0
- clue_api-1.0.0.dev7/clue/services/fetcher_service.py +203 -0
- clue_api-1.0.0.dev7/clue/services/jwt_service.py +233 -0
- clue_api-1.0.0.dev7/clue/services/lookup_service.py +786 -0
- clue_api-1.0.0.dev7/clue/services/type_service.py +165 -0
- clue_api-1.0.0.dev7/clue/services/user_service.py +152 -0
- clue_api-1.0.0.dev7/pyproject.toml +288 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 Crown Copyright, Government of Canada (Canadian Centre for Cyber Security / Communications Security Establishment)
|
|
4
|
+
|
|
5
|
+
Copyright title to all 3rd party software distributed with Clue is held by the respective copyright holders as noted in those files. Users are asked to read the 3rd Party Licenses referenced with those assets.
|
|
6
|
+
|
|
7
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
8
|
+
|
|
9
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
10
|
+
|
|
11
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: clue-api
|
|
3
|
+
Version: 1.0.0.dev7
|
|
4
|
+
Summary: Clue distributed enrichment service
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: clue,distributed,enrichment,gc,canada,cse-cst,cse,cst,cyber,cccs
|
|
8
|
+
Author: Canadian Centre for Cyber Security
|
|
9
|
+
Author-email: contact@cyber.gc.ca
|
|
10
|
+
Requires-Python: >=3.12,<4.0
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Provides-Extra: server
|
|
20
|
+
Requires-Dist: PyYAML (>=6.0.1,<7.0.0) ; extra == "server"
|
|
21
|
+
Requires-Dist: Werkzeug (>=3.0.2,<4.0.0) ; extra == "server"
|
|
22
|
+
Requires-Dist: apscheduler (>=3.10.4,<4.0.0) ; extra == "server"
|
|
23
|
+
Requires-Dist: authlib (<1.0.0) ; extra == "server"
|
|
24
|
+
Requires-Dist: bcrypt (>=4.1.2,<5.0.0) ; extra == "server"
|
|
25
|
+
Requires-Dist: beautifulsoup4 (>=4.13.3,<5.0.0)
|
|
26
|
+
Requires-Dist: cart (>=1.2.3,<2.0.0)
|
|
27
|
+
Requires-Dist: elastic-apm (>=6.22.0,<7.0.0)
|
|
28
|
+
Requires-Dist: flasgger (>=0.9.7.1,<0.10.0.0) ; extra == "server"
|
|
29
|
+
Requires-Dist: flask (<3.0.0)
|
|
30
|
+
Requires-Dist: flask-caching (>=2.1.0,<3.0.0)
|
|
31
|
+
Requires-Dist: flask-cors (>=4.0.1,<5.0.0) ; extra == "server"
|
|
32
|
+
Requires-Dist: gevent (>=24.2.1,<25.0.0)
|
|
33
|
+
Requires-Dist: geventhttpclient (>=2.3.1,<3.0.0)
|
|
34
|
+
Requires-Dist: gunicorn (>=22.0.0,<23.0.0)
|
|
35
|
+
Requires-Dist: imgkit (>=1.2.3,<2.0.0)
|
|
36
|
+
Requires-Dist: netifaces (>=0.11.0,<0.12.0) ; extra == "server"
|
|
37
|
+
Requires-Dist: passlib (>=1.7.4,<2.0.0) ; extra == "server"
|
|
38
|
+
Requires-Dist: pillow (>=11.1.0,<12.0.0)
|
|
39
|
+
Requires-Dist: prometheus-client (>=0.20.0,<0.21.0) ; extra == "server"
|
|
40
|
+
Requires-Dist: pydantic (>=2.7.1,<3.0.0)
|
|
41
|
+
Requires-Dist: pydantic-settings[yaml] (>=2.3.4,<3.0.0)
|
|
42
|
+
Requires-Dist: pyjwt (>=2.8.0,<3.0.0) ; extra == "server"
|
|
43
|
+
Requires-Dist: pyroute2 (>=0.7.12,<0.8.0) ; extra == "server"
|
|
44
|
+
Requires-Dist: python-baseconv (>=1.2.2,<2.0.0) ; extra == "server"
|
|
45
|
+
Requires-Dist: pytz (>=2024.1,<2025.0) ; extra == "server"
|
|
46
|
+
Requires-Dist: redis (>=5.0.3,<6.0.0)
|
|
47
|
+
Requires-Dist: requests (>=2.32.5,<3.0.0)
|
|
48
|
+
Requires-Dist: setuptools (<78.0.0)
|
|
49
|
+
Requires-Dist: trino (>=0.336.0,<0.337.0)
|
|
50
|
+
Project-URL: Documentation, https://github.com/CybercentreCanada/clue
|
|
51
|
+
Project-URL: Homepage, https://github.com/CybercentreCanada/clue
|
|
52
|
+
Project-URL: Repository, https://github.com/CybercentreCanada/clue
|
|
53
|
+
Description-Content-Type: text/markdown
|
|
54
|
+
|
|
55
|
+
# Clue
|
|
56
|
+
|
|
57
|
+
To start the API for clue, check to ensure that:
|
|
58
|
+
|
|
59
|
+
1. Docker is composed up through `dev/docker-compose.yml`
|
|
60
|
+
1. Note that you may need to set up uchimera container connections if you have not tyet done so:
|
|
61
|
+
2. `az login && az acr login -n uchimera`
|
|
62
|
+
3. If you do not have permission, reach out to APA2B.
|
|
63
|
+
2. `cd clue/api`
|
|
64
|
+
3. Run `poetry install` within the clue/api folder to install all dependencies
|
|
65
|
+
4. You may need to run `poetry install --with test,dev,types,plugins --all-extras`
|
|
66
|
+
5. Run `sudo mkdir -p /var/log/clue/`
|
|
67
|
+
6. Run `sudo mkdir -p /etc/clue/conf/`
|
|
68
|
+
7. Run `sudo chmod a+rw /var/log/clue/`
|
|
69
|
+
8. Run `sudo chmod a+rw /etc/clue/conf/`
|
|
70
|
+
9. Run `cp build_scripts/classification.yml /etc/clue/conf/classification.yml`
|
|
71
|
+
10. Run `cp test/unit/config.yml /etc/clue/conf/config.yml`
|
|
72
|
+
11. To start server: `poetry run server`
|
|
73
|
+
|
|
74
|
+
To start Enrichment Testing:
|
|
75
|
+
|
|
76
|
+
* In order to have the local server connect to the UI the servers need to be ran manually
|
|
77
|
+
* Please ensure that ```pwd``` is clue/api
|
|
78
|
+
* May need to add ```poetry run``` before each command
|
|
79
|
+
|
|
80
|
+
1. ```flask --app test.utils.test_server run --no-reload --port 5008```
|
|
81
|
+
2. ```flask --app test.utils.bad_server run --no-reload --port 5009```
|
|
82
|
+
3. ```flask --app test.utils.slow_server run --no-reload --port 5010```
|
|
83
|
+
4. ```flask --app test.utils.telemetry_server run --no-reload --port 5011```
|
|
84
|
+
|
|
85
|
+
Troubleshooting:
|
|
86
|
+
|
|
87
|
+
1. If there are issues with these steps please check the build system for poetry installation steps
|
|
88
|
+
2. The scripts will show all necessary directories that need to be made in order for classfication to work
|
|
89
|
+
|
|
90
|
+
## Contributing
|
|
91
|
+
|
|
92
|
+
See [CONTRIBUTING.md](documentation/CONTRIBUTING.md) for more information
|
|
93
|
+
|
|
94
|
+
## FAQ
|
|
95
|
+
|
|
96
|
+
### I'm getting permissions issues on `/var/log/clue` or `/etc/clue/conf`?
|
|
97
|
+
|
|
98
|
+
Run `sudo chmod a+rw /var/log/clue/` and `sudo chmod a+rw /etc/clue/conf/`.
|
|
99
|
+
|
|
100
|
+
### How can I add dependencies for my plugin?
|
|
101
|
+
|
|
102
|
+
See [this section](documentation/CONTRIBUTING.md#external-dependencies) of CONTRIBUTING.md.
|
|
103
|
+
|
|
104
|
+
### Email rendering does not seem to be working?
|
|
105
|
+
|
|
106
|
+
You must install `wkhtmltopdf`, both locally for development and in your Dockerfile:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
sudo apt install wkhtmltopdf
|
|
110
|
+
```
|
|
111
|
+
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Clue
|
|
2
|
+
|
|
3
|
+
To start the API for clue, check to ensure that:
|
|
4
|
+
|
|
5
|
+
1. Docker is composed up through `dev/docker-compose.yml`
|
|
6
|
+
1. Note that you may need to set up uchimera container connections if you have not tyet done so:
|
|
7
|
+
2. `az login && az acr login -n uchimera`
|
|
8
|
+
3. If you do not have permission, reach out to APA2B.
|
|
9
|
+
2. `cd clue/api`
|
|
10
|
+
3. Run `poetry install` within the clue/api folder to install all dependencies
|
|
11
|
+
4. You may need to run `poetry install --with test,dev,types,plugins --all-extras`
|
|
12
|
+
5. Run `sudo mkdir -p /var/log/clue/`
|
|
13
|
+
6. Run `sudo mkdir -p /etc/clue/conf/`
|
|
14
|
+
7. Run `sudo chmod a+rw /var/log/clue/`
|
|
15
|
+
8. Run `sudo chmod a+rw /etc/clue/conf/`
|
|
16
|
+
9. Run `cp build_scripts/classification.yml /etc/clue/conf/classification.yml`
|
|
17
|
+
10. Run `cp test/unit/config.yml /etc/clue/conf/config.yml`
|
|
18
|
+
11. To start server: `poetry run server`
|
|
19
|
+
|
|
20
|
+
To start Enrichment Testing:
|
|
21
|
+
|
|
22
|
+
* In order to have the local server connect to the UI the servers need to be ran manually
|
|
23
|
+
* Please ensure that ```pwd``` is clue/api
|
|
24
|
+
* May need to add ```poetry run``` before each command
|
|
25
|
+
|
|
26
|
+
1. ```flask --app test.utils.test_server run --no-reload --port 5008```
|
|
27
|
+
2. ```flask --app test.utils.bad_server run --no-reload --port 5009```
|
|
28
|
+
3. ```flask --app test.utils.slow_server run --no-reload --port 5010```
|
|
29
|
+
4. ```flask --app test.utils.telemetry_server run --no-reload --port 5011```
|
|
30
|
+
|
|
31
|
+
Troubleshooting:
|
|
32
|
+
|
|
33
|
+
1. If there are issues with these steps please check the build system for poetry installation steps
|
|
34
|
+
2. The scripts will show all necessary directories that need to be made in order for classfication to work
|
|
35
|
+
|
|
36
|
+
## Contributing
|
|
37
|
+
|
|
38
|
+
See [CONTRIBUTING.md](documentation/CONTRIBUTING.md) for more information
|
|
39
|
+
|
|
40
|
+
## FAQ
|
|
41
|
+
|
|
42
|
+
### I'm getting permissions issues on `/var/log/clue` or `/etc/clue/conf`?
|
|
43
|
+
|
|
44
|
+
Run `sudo chmod a+rw /var/log/clue/` and `sudo chmod a+rw /etc/clue/conf/`.
|
|
45
|
+
|
|
46
|
+
### How can I add dependencies for my plugin?
|
|
47
|
+
|
|
48
|
+
See [this section](documentation/CONTRIBUTING.md#external-dependencies) of CONTRIBUTING.md.
|
|
49
|
+
|
|
50
|
+
### Email rendering does not seem to be working?
|
|
51
|
+
|
|
52
|
+
You must install `wkhtmltopdf`, both locally for development and in your Dockerfile:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
sudo apt install wkhtmltopdf
|
|
56
|
+
```
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Add any directories, files, or patterns you don't want to be tracked by version control
|
|
2
|
+
|
|
3
|
+
# IDE files
|
|
4
|
+
.pydevproject
|
|
5
|
+
.python-version
|
|
6
|
+
.idea
|
|
7
|
+
|
|
8
|
+
# Testing artifacts
|
|
9
|
+
.pytest_cache
|
|
10
|
+
htmlcov
|
|
11
|
+
.coverage
|
|
12
|
+
|
|
13
|
+
# OS Files
|
|
14
|
+
ehthumbs.db
|
|
15
|
+
Thumbs.db
|
|
16
|
+
|
|
17
|
+
# Build artifacts
|
|
18
|
+
build/
|
|
19
|
+
dist/
|
|
20
|
+
*.py[cod]
|
|
21
|
+
*.egg-info
|
|
File without changes
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
from sys import exc_info
|
|
2
|
+
from traceback import format_tb
|
|
3
|
+
from typing import Any, Union
|
|
4
|
+
|
|
5
|
+
from flask import Blueprint, Response, make_response, request
|
|
6
|
+
from prometheus_client import Counter
|
|
7
|
+
|
|
8
|
+
from clue.common.forge import APP_NAME
|
|
9
|
+
from clue.common.logging import get_logger, log_with_traceback
|
|
10
|
+
from clue.common.str_utils import safe_str
|
|
11
|
+
from clue.models.network import ClueResponse
|
|
12
|
+
|
|
13
|
+
API_PREFIX = "/api"
|
|
14
|
+
RAW_API_COUNTER = Counter(
|
|
15
|
+
f"{APP_NAME.replace('-', '_')}_http_requests_total", # type: ignore[union-attr]
|
|
16
|
+
"HTTP Requests broken down by method, path, and status",
|
|
17
|
+
["method", "path", "status"],
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
logger = get_logger(__file__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def make_subapi_blueprint(name, api_version=1):
|
|
24
|
+
"""Create a flask Blueprint for a subapi in a standard way."""
|
|
25
|
+
return Blueprint(name, name, url_prefix="/".join([API_PREFIX, f"v{api_version}", name]))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _make_api_response(
|
|
29
|
+
data: Any, err: Union[str, Exception] = "", warnings: list[str] = [], status_code: int = 200, cookies: Any = None
|
|
30
|
+
) -> Response:
|
|
31
|
+
if isinstance(err, Exception): # pragma: no cover
|
|
32
|
+
trace = exc_info()[2]
|
|
33
|
+
err = "".join(["\n"] + format_tb(trace) + ["%s: %s\n" % (err.__class__.__name__, str(err))]).rstrip("\n")
|
|
34
|
+
log_with_traceback(trace, "Exception", is_exception=True)
|
|
35
|
+
|
|
36
|
+
resp = make_response(
|
|
37
|
+
ClueResponse(response=data, error_message=err, warning=warnings, status_code=status_code).model_dump(
|
|
38
|
+
mode="json", by_alias=True, exclude_none=True
|
|
39
|
+
),
|
|
40
|
+
status_code,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
resp.headers["Content-Type"] = "application/json"
|
|
44
|
+
|
|
45
|
+
if isinstance(cookies, dict):
|
|
46
|
+
for k, v in cookies.items():
|
|
47
|
+
resp.set_cookie(k, v)
|
|
48
|
+
|
|
49
|
+
RAW_API_COUNTER.labels(request.method, str(request.url_rule), status_code).inc()
|
|
50
|
+
logger.info("%s %s - %s", request.method, request.path, status_code)
|
|
51
|
+
|
|
52
|
+
return resp
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# Some helper functions for make_api_response
|
|
56
|
+
|
|
57
|
+
DEFAULT_DATA = {True: {"success": True}, False: {"success": False}}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def ok(data=DEFAULT_DATA[True], cookies=None):
|
|
61
|
+
"""Returns response with status code 200"""
|
|
62
|
+
return _make_api_response(data, status_code=200, cookies=cookies)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def created(data=DEFAULT_DATA[True], warnings=[], cookies=None):
|
|
66
|
+
"""Returns response with status code 201"""
|
|
67
|
+
return _make_api_response(data, warnings=warnings, status_code=201, cookies=cookies)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def accepted(data=DEFAULT_DATA[True], cookies=None):
|
|
71
|
+
"""Returns response with status code 202"""
|
|
72
|
+
return _make_api_response(data, status_code=202, cookies=cookies)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def no_content(data=None, cookies=None):
|
|
76
|
+
"""Returns response with status code 204"""
|
|
77
|
+
return _make_api_response(data or DEFAULT_DATA[True], status_code=204, cookies=cookies)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def not_modified(data=DEFAULT_DATA[True], cookies=None):
|
|
81
|
+
"""Returns response with status code 304"""
|
|
82
|
+
return _make_api_response(data, status_code=304, cookies=cookies)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def bad_request(data=DEFAULT_DATA[False], err="", cookies=None, warnings=[]):
|
|
86
|
+
"""Returns response with status code ies"""
|
|
87
|
+
return _make_api_response(data, err, status_code=400, cookies=cookies, warnings=warnings)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def unauthorized(data=DEFAULT_DATA[False], err="", cookies=None):
|
|
91
|
+
"""Returns response with status code 401"""
|
|
92
|
+
return _make_api_response(data, err, status_code=401, cookies=cookies)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def forbidden(data=DEFAULT_DATA[False], err="", cookies=None):
|
|
96
|
+
"""Returns response with status code 403"""
|
|
97
|
+
return _make_api_response(data, err, status_code=403, cookies=cookies)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def not_found(data=DEFAULT_DATA[False], err="", cookies=None):
|
|
101
|
+
"""Returns response with status code 404"""
|
|
102
|
+
return _make_api_response(data, err, status_code=404, cookies=cookies)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def conflict(data=DEFAULT_DATA[False], err="", cookies=None):
|
|
106
|
+
"""Returns response with status code 409"""
|
|
107
|
+
return _make_api_response(data, err, status_code=409, cookies=cookies)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def precondition_failed(data=DEFAULT_DATA[False], err="", cookies=None):
|
|
111
|
+
"""Returns response with status code 412"""
|
|
112
|
+
return _make_api_response(data, err, status_code=412, cookies=cookies)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def teapot(data={**DEFAULT_DATA[False], "teapot": True}, err="", cookies=None):
|
|
116
|
+
"""Returns response with status code 418"""
|
|
117
|
+
return _make_api_response(data, err, status_code=418, cookies=cookies)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def too_many_requests(data=DEFAULT_DATA[False], err="", cookies=None):
|
|
121
|
+
"""Returns response with status code 429"""
|
|
122
|
+
return _make_api_response(data, err, status_code=429, cookies=cookies)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def internal_error(
|
|
126
|
+
data={**DEFAULT_DATA[False]},
|
|
127
|
+
err="Something went wrong. Contact an administrator.",
|
|
128
|
+
cookies=None,
|
|
129
|
+
):
|
|
130
|
+
"""Returns response with status code 500"""
|
|
131
|
+
return _make_api_response(data, err, status_code=500, cookies=cookies)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def not_implemented(
|
|
135
|
+
data={**DEFAULT_DATA[False]},
|
|
136
|
+
err="Something went wrong. Contact an administrator.",
|
|
137
|
+
cookies=None,
|
|
138
|
+
):
|
|
139
|
+
"""Returns response with status code 501"""
|
|
140
|
+
return _make_api_response(data, err, status_code=501, cookies=cookies)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def bad_gateway(
|
|
144
|
+
data={**DEFAULT_DATA[False]},
|
|
145
|
+
err="Something went wrong. Contact an administrator.",
|
|
146
|
+
cookies=None,
|
|
147
|
+
):
|
|
148
|
+
"""Returns response with status code 502"""
|
|
149
|
+
return _make_api_response(data, err, status_code=502, cookies=cookies)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def service_unavailable(
|
|
153
|
+
data={**DEFAULT_DATA[False]},
|
|
154
|
+
err="Something went wrong. Contact an administrator.",
|
|
155
|
+
cookies=None,
|
|
156
|
+
):
|
|
157
|
+
"""Returns response with status code 503"""
|
|
158
|
+
return _make_api_response(data, err, status_code=503, cookies=cookies)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def make_file_response(data, name, size, status_code=200, content_type="application/octet-stream"):
|
|
162
|
+
"""Returns file response with arbitrary status code"""
|
|
163
|
+
response = make_response(data, status_code)
|
|
164
|
+
response.headers["Content-Type"] = content_type
|
|
165
|
+
response.headers["Content-Length"] = size
|
|
166
|
+
response.headers["Content-Disposition"] = 'attachment; filename="%s"' % safe_str(name)
|
|
167
|
+
return response
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def stream_file_response(reader, name, size, status_code=200):
|
|
171
|
+
"""Returns stream response with arbitrary status code"""
|
|
172
|
+
chunk_size = 65535
|
|
173
|
+
|
|
174
|
+
def generate():
|
|
175
|
+
reader.seek(0)
|
|
176
|
+
while True:
|
|
177
|
+
data = reader.read(chunk_size)
|
|
178
|
+
if not data:
|
|
179
|
+
break
|
|
180
|
+
yield data
|
|
181
|
+
reader.close()
|
|
182
|
+
|
|
183
|
+
headers = {
|
|
184
|
+
"Content-Type": "application/octet-stream",
|
|
185
|
+
"Content-Length": size,
|
|
186
|
+
"Content-Disposition": 'attachment; filename="%s"' % safe_str(name),
|
|
187
|
+
}
|
|
188
|
+
return Response(generate(), status=status_code, headers=headers)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def make_binary_response(data, size, status_code=200):
|
|
192
|
+
"""Returns binary response with arbitrary status code"""
|
|
193
|
+
response = make_response(data, status_code)
|
|
194
|
+
response.headers["Content-Type"] = "application/octet-stream"
|
|
195
|
+
response.headers["Content-Length"] = size
|
|
196
|
+
return response
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def stream_binary_response(reader, status_code=200):
|
|
200
|
+
"""Returns streamed binary response with arbitrary status code"""
|
|
201
|
+
chunk_size = 4096
|
|
202
|
+
|
|
203
|
+
def generate():
|
|
204
|
+
reader.seek(0)
|
|
205
|
+
while True:
|
|
206
|
+
data = reader.read(chunk_size)
|
|
207
|
+
if not data:
|
|
208
|
+
break
|
|
209
|
+
yield data
|
|
210
|
+
|
|
211
|
+
return Response(generate(), status=status_code, mimetype="application/octet-stream")
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from flask import Blueprint, current_app, request
|
|
2
|
+
|
|
3
|
+
from clue.api import ok
|
|
4
|
+
from clue.common.logging import get_logger
|
|
5
|
+
from clue.security import api_login
|
|
6
|
+
|
|
7
|
+
logger = get_logger(__file__)
|
|
8
|
+
|
|
9
|
+
API_PREFIX = "/api"
|
|
10
|
+
api = Blueprint("api", __name__, url_prefix=API_PREFIX)
|
|
11
|
+
|
|
12
|
+
XSRF_ENABLED = True
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
#####################################
|
|
16
|
+
# API list API (API inception)
|
|
17
|
+
@api.route("/")
|
|
18
|
+
@api_login(audit=False)
|
|
19
|
+
def api_version_list(**_):
|
|
20
|
+
"""List all available API versions.
|
|
21
|
+
|
|
22
|
+
Variables:
|
|
23
|
+
None
|
|
24
|
+
|
|
25
|
+
Arguments:
|
|
26
|
+
None
|
|
27
|
+
|
|
28
|
+
Data Block:
|
|
29
|
+
None
|
|
30
|
+
|
|
31
|
+
Result example:
|
|
32
|
+
["v1", "v2", "v3"] #List of API versions available
|
|
33
|
+
"""
|
|
34
|
+
api_list = []
|
|
35
|
+
for rule in current_app.url_map.iter_rules():
|
|
36
|
+
if rule.rule.startswith("/api/"):
|
|
37
|
+
version = rule.rule[5:].split("/", 1)[0]
|
|
38
|
+
if version not in api_list and version != "":
|
|
39
|
+
# noinspection PyBroadException
|
|
40
|
+
try:
|
|
41
|
+
int(version[1:])
|
|
42
|
+
except ValueError:
|
|
43
|
+
continue
|
|
44
|
+
api_list.append(version)
|
|
45
|
+
|
|
46
|
+
return ok(api_list)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@api.route("/site_map/")
|
|
50
|
+
@api_login(audit=False)
|
|
51
|
+
def site_map(**_):
|
|
52
|
+
"""Check if all pages have been protected by a login decorator
|
|
53
|
+
|
|
54
|
+
Variables:
|
|
55
|
+
None
|
|
56
|
+
|
|
57
|
+
Arguments:
|
|
58
|
+
unsafe_only => Only show unsafe pages
|
|
59
|
+
|
|
60
|
+
Data Block:
|
|
61
|
+
None
|
|
62
|
+
|
|
63
|
+
Result example:
|
|
64
|
+
[ #List of pages dictionary containing...
|
|
65
|
+
{"function": views.default, #Function name
|
|
66
|
+
"url": "/", #Url to page
|
|
67
|
+
"protected": true, #Is function login protected
|
|
68
|
+
"methods": ["GET"]}, #Methods allowed to access the page
|
|
69
|
+
]
|
|
70
|
+
"""
|
|
71
|
+
pages = []
|
|
72
|
+
for rule in current_app.url_map.iter_rules():
|
|
73
|
+
func = current_app.view_functions[rule.endpoint]
|
|
74
|
+
methods = []
|
|
75
|
+
if rule.methods:
|
|
76
|
+
for item in rule.methods:
|
|
77
|
+
if item != "OPTIONS" and item != "HEAD":
|
|
78
|
+
methods.append(item)
|
|
79
|
+
protected = func.__dict__.get("protected", False)
|
|
80
|
+
audit = func.__dict__.get("audit", False)
|
|
81
|
+
if "/api/v1/" in rule.rule:
|
|
82
|
+
prefix = "api.v1."
|
|
83
|
+
else:
|
|
84
|
+
prefix = ""
|
|
85
|
+
|
|
86
|
+
if "unsafe_only" in request.args and protected:
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
pages.append(
|
|
90
|
+
{
|
|
91
|
+
"function": f"{prefix}{rule.endpoint.replace('apiv1.', '')}",
|
|
92
|
+
"url": rule.rule,
|
|
93
|
+
"methods": methods,
|
|
94
|
+
"protected": protected,
|
|
95
|
+
"audit": audit,
|
|
96
|
+
}
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
return ok(sorted(pages, key=lambda i: i["url"]))
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from textwrap import dedent
|
|
2
|
+
|
|
3
|
+
from flask import Blueprint, current_app, request
|
|
4
|
+
|
|
5
|
+
from clue.api import ok
|
|
6
|
+
from clue.api.base import api_login
|
|
7
|
+
|
|
8
|
+
API_PREFIX = "/api/v1"
|
|
9
|
+
apiv1 = Blueprint("apiv1", __name__, url_prefix=API_PREFIX)
|
|
10
|
+
apiv1._doc = "Api Documentation - Verison 1" # type: ignore[attr-defined]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
#####################################
|
|
14
|
+
# API DOCUMENTATION
|
|
15
|
+
# noinspection PyProtectedMember,PyBroadException
|
|
16
|
+
@apiv1.route("/")
|
|
17
|
+
@api_login(audit=False)
|
|
18
|
+
def get_api_documentation(**_):
|
|
19
|
+
"""Full API doc. Loop through all registered API paths and display their documentation.
|
|
20
|
+
|
|
21
|
+
Returns a list of API definition.
|
|
22
|
+
|
|
23
|
+
Variables:
|
|
24
|
+
None
|
|
25
|
+
|
|
26
|
+
Arguments:
|
|
27
|
+
None
|
|
28
|
+
|
|
29
|
+
Result Example:
|
|
30
|
+
[
|
|
31
|
+
{'name': "Api Doc", # Name of the api
|
|
32
|
+
'path': "/api/path/<variable>/", # API path
|
|
33
|
+
'methods': ["GET", "POST"], # Allowed HTTP methods
|
|
34
|
+
'description': "API doc.", # API documentation
|
|
35
|
+
'id': "api_doc", # Unique ID for the API
|
|
36
|
+
'function': "apiv1.api_doc", # Function called in the code
|
|
37
|
+
'protected': False, # Does the API require login?
|
|
38
|
+
'complete' : True}, # Is the API stable?
|
|
39
|
+
...]
|
|
40
|
+
"""
|
|
41
|
+
api_blueprints = {}
|
|
42
|
+
api_list = []
|
|
43
|
+
for rule in current_app.url_map.iter_rules():
|
|
44
|
+
if rule.rule.startswith(request.path):
|
|
45
|
+
methods = [item for item in (rule.methods or []) if item != "OPTIONS" and item != "HEAD"]
|
|
46
|
+
|
|
47
|
+
func = current_app.view_functions[rule.endpoint]
|
|
48
|
+
doc_string = func.__doc__
|
|
49
|
+
func_title = " ".join([x.capitalize() for x in rule.endpoint[rule.endpoint.rindex(".") + 1 :].split("_")])
|
|
50
|
+
blueprint = rule.endpoint[: rule.endpoint.rindex(".")]
|
|
51
|
+
if blueprint == "apiv1":
|
|
52
|
+
blueprint = "documentation"
|
|
53
|
+
|
|
54
|
+
if blueprint not in api_blueprints:
|
|
55
|
+
try:
|
|
56
|
+
doc = current_app.blueprints[rule.endpoint[: rule.endpoint.rindex(".")]]._doc # type: ignore[attr-defined]
|
|
57
|
+
except Exception:
|
|
58
|
+
doc = ""
|
|
59
|
+
|
|
60
|
+
api_blueprints[blueprint] = doc
|
|
61
|
+
|
|
62
|
+
if doc_string:
|
|
63
|
+
description = dedent(doc_string)
|
|
64
|
+
else:
|
|
65
|
+
description = "[INCOMPLETE]\n\nTHIS API HAS NOT BEEN DOCUMENTED YET!"
|
|
66
|
+
|
|
67
|
+
api_id = rule.endpoint.replace("apiv1.", "").replace(".", "_")
|
|
68
|
+
|
|
69
|
+
api_list.append(
|
|
70
|
+
{
|
|
71
|
+
"protected": func.__dict__.get("protected", False),
|
|
72
|
+
"name": func_title,
|
|
73
|
+
"id": api_id,
|
|
74
|
+
"function": f"api.v1.{rule.endpoint}",
|
|
75
|
+
"path": rule.rule,
|
|
76
|
+
"methods": methods,
|
|
77
|
+
"description": description,
|
|
78
|
+
"complete": "[INCOMPLETE]" not in description,
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
return ok({"apis": api_list, "blueprints": api_blueprints})
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Enrichment Actions
|
|
2
|
+
|
|
3
|
+
List and execute actions
|
|
4
|
+
|
|
5
|
+
* Provide endpoints to list valid actions exposed by plugins.
|
|
6
|
+
* Provide endpoints to execute these actions.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from flask_cors import CORS
|
|
10
|
+
|
|
11
|
+
from clue.api import internal_error, make_subapi_blueprint, not_found, ok
|
|
12
|
+
from clue.common.exceptions import ClueException, NotFoundException
|
|
13
|
+
from clue.common.logging import get_logger
|
|
14
|
+
from clue.common.swagger import generate_swagger_docs
|
|
15
|
+
from clue.config import config
|
|
16
|
+
from clue.models.actions import Action, ActionResult
|
|
17
|
+
from clue.security import api_login
|
|
18
|
+
from clue.services import action_service
|
|
19
|
+
|
|
20
|
+
logger = get_logger(__file__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
SUB_API = "actions"
|
|
24
|
+
actions_api = make_subapi_blueprint(SUB_API, api_version=1)
|
|
25
|
+
actions_api._doc = "Run actions on data through configured external data sources/systems."
|
|
26
|
+
|
|
27
|
+
CORS(actions_api, origins=config.ui.cors_origins, supports_credentials=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@generate_swagger_docs(responses={200: "A list of types and their classification"})
|
|
31
|
+
@actions_api.route("/", methods=["GET"])
|
|
32
|
+
@api_login()
|
|
33
|
+
def get_actions(**kwargs) -> dict[str, Action]:
|
|
34
|
+
"""Return the supported actions of each external service.
|
|
35
|
+
|
|
36
|
+
Variables:
|
|
37
|
+
None
|
|
38
|
+
|
|
39
|
+
Arguments:
|
|
40
|
+
None
|
|
41
|
+
|
|
42
|
+
Result Example:
|
|
43
|
+
{ # A dictionary of sources with their supported actions.
|
|
44
|
+
<source_id>.<action_id>: {
|
|
45
|
+
"id": "",
|
|
46
|
+
"name": "",
|
|
47
|
+
"classification": "",
|
|
48
|
+
"summary": "",
|
|
49
|
+
"supported_types": "",
|
|
50
|
+
"params": {
|
|
51
|
+
<JSON schema>
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
...,
|
|
55
|
+
}
|
|
56
|
+
"""
|
|
57
|
+
return ok(action_service.get_plugins_supported_actions(kwargs["user"]))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@generate_swagger_docs(responses={200: "Successful lookup to selected plugins"})
|
|
61
|
+
@actions_api.route("/execute/<plugin_id>/<action_id>", methods=["POST"])
|
|
62
|
+
@api_login()
|
|
63
|
+
def execute_action(plugin_id: str, action_id: str, **kwargs) -> ActionResult:
|
|
64
|
+
"""Search other services for additional information related to the provided data.
|
|
65
|
+
|
|
66
|
+
Variables:
|
|
67
|
+
plugin_id (str): the ID of the plugin who owns the action to execute
|
|
68
|
+
action_id (str): the ID of the action to execute
|
|
69
|
+
|
|
70
|
+
Arguments:
|
|
71
|
+
None
|
|
72
|
+
|
|
73
|
+
Data Block:
|
|
74
|
+
{
|
|
75
|
+
type: "ip",
|
|
76
|
+
value: "127.0.0.1",
|
|
77
|
+
...
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
Result Example:
|
|
81
|
+
{
|
|
82
|
+
"outcome": "success | failure", # was this execution a success or failure?
|
|
83
|
+
"format": "link", # What format is the output in?
|
|
84
|
+
"output": "http://example.com" # The output of the action. Can be any data structure.
|
|
85
|
+
}
|
|
86
|
+
"""
|
|
87
|
+
try:
|
|
88
|
+
return ok(action_service.execute_action(plugin_id, action_id, kwargs["user"]))
|
|
89
|
+
except NotFoundException as err:
|
|
90
|
+
return not_found(err=err.message)
|
|
91
|
+
except ClueException as err:
|
|
92
|
+
return internal_error(err=err.message)
|