clue-api 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. clue_api-1.0.0/LICENSE +11 -0
  2. clue_api-1.0.0/PKG-INFO +110 -0
  3. clue_api-1.0.0/README.md +56 -0
  4. clue_api-1.0.0/clue/.gitignore +21 -0
  5. clue_api-1.0.0/clue/__init__.py +0 -0
  6. clue_api-1.0.0/clue/api/__init__.py +211 -0
  7. clue_api-1.0.0/clue/api/base.py +99 -0
  8. clue_api-1.0.0/clue/api/v1/__init__.py +82 -0
  9. clue_api-1.0.0/clue/api/v1/actions.py +92 -0
  10. clue_api-1.0.0/clue/api/v1/auth.py +243 -0
  11. clue_api-1.0.0/clue/api/v1/configs.py +83 -0
  12. clue_api-1.0.0/clue/api/v1/fetchers.py +94 -0
  13. clue_api-1.0.0/clue/api/v1/lookup.py +221 -0
  14. clue_api-1.0.0/clue/api/v1/registration.py +109 -0
  15. clue_api-1.0.0/clue/api/v1/static.py +94 -0
  16. clue_api-1.0.0/clue/app.py +166 -0
  17. clue_api-1.0.0/clue/cache/__init__.py +129 -0
  18. clue_api-1.0.0/clue/common/__init__.py +0 -0
  19. clue_api-1.0.0/clue/common/classification.py +1006 -0
  20. clue_api-1.0.0/clue/common/classification.yml +130 -0
  21. clue_api-1.0.0/clue/common/dict_utils.py +130 -0
  22. clue_api-1.0.0/clue/common/exceptions.py +199 -0
  23. clue_api-1.0.0/clue/common/forge.py +152 -0
  24. clue_api-1.0.0/clue/common/json_utils.py +10 -0
  25. clue_api-1.0.0/clue/common/list_utils.py +11 -0
  26. clue_api-1.0.0/clue/common/logging/__init__.py +291 -0
  27. clue_api-1.0.0/clue/common/logging/audit.py +157 -0
  28. clue_api-1.0.0/clue/common/logging/format.py +42 -0
  29. clue_api-1.0.0/clue/common/regex.py +31 -0
  30. clue_api-1.0.0/clue/common/str_utils.py +213 -0
  31. clue_api-1.0.0/clue/common/swagger.py +139 -0
  32. clue_api-1.0.0/clue/common/uid.py +47 -0
  33. clue_api-1.0.0/clue/config.py +60 -0
  34. clue_api-1.0.0/clue/constants/__init__.py +0 -0
  35. clue_api-1.0.0/clue/constants/supported_types.py +38 -0
  36. clue_api-1.0.0/clue/cronjobs/__init__.py +30 -0
  37. clue_api-1.0.0/clue/cronjobs/plugins.py +32 -0
  38. clue_api-1.0.0/clue/error.py +129 -0
  39. clue_api-1.0.0/clue/gunicorn_config.py +29 -0
  40. clue_api-1.0.0/clue/healthz.py +74 -0
  41. clue_api-1.0.0/clue/helper/discover.py +53 -0
  42. clue_api-1.0.0/clue/helper/headers.py +30 -0
  43. clue_api-1.0.0/clue/helper/oauth.py +128 -0
  44. clue_api-1.0.0/clue/models/__init__.py +0 -0
  45. clue_api-1.0.0/clue/models/actions.py +243 -0
  46. clue_api-1.0.0/clue/models/config.py +455 -0
  47. clue_api-1.0.0/clue/models/fetchers.py +136 -0
  48. clue_api-1.0.0/clue/models/graph.py +162 -0
  49. clue_api-1.0.0/clue/models/model_list.py +52 -0
  50. clue_api-1.0.0/clue/models/network.py +430 -0
  51. clue_api-1.0.0/clue/models/results/__init__.py +34 -0
  52. clue_api-1.0.0/clue/models/results/base.py +10 -0
  53. clue_api-1.0.0/clue/models/results/graph.py +26 -0
  54. clue_api-1.0.0/clue/models/results/image.py +22 -0
  55. clue_api-1.0.0/clue/models/results/status.py +55 -0
  56. clue_api-1.0.0/clue/models/results/validation.py +57 -0
  57. clue_api-1.0.0/clue/models/selector.py +67 -0
  58. clue_api-1.0.0/clue/models/utils.py +52 -0
  59. clue_api-1.0.0/clue/models/validators.py +19 -0
  60. clue_api-1.0.0/clue/patched.py +8 -0
  61. clue_api-1.0.0/clue/plugin/__init__.py +1008 -0
  62. clue_api-1.0.0/clue/plugin/helpers/__init__.py +0 -0
  63. clue_api-1.0.0/clue/plugin/helpers/central_server.py +27 -0
  64. clue_api-1.0.0/clue/plugin/helpers/email_render.py +228 -0
  65. clue_api-1.0.0/clue/plugin/helpers/token.py +34 -0
  66. clue_api-1.0.0/clue/plugin/helpers/trino.py +103 -0
  67. clue_api-1.0.0/clue/plugin/interactive.py +262 -0
  68. clue_api-1.0.0/clue/plugin/models.py +19 -0
  69. clue_api-1.0.0/clue/plugin/utils.py +78 -0
  70. clue_api-1.0.0/clue/remote/__init__.py +0 -0
  71. clue_api-1.0.0/clue/remote/datatypes/__init__.py +130 -0
  72. clue_api-1.0.0/clue/remote/datatypes/cache.py +62 -0
  73. clue_api-1.0.0/clue/remote/datatypes/events.py +118 -0
  74. clue_api-1.0.0/clue/remote/datatypes/hash.py +193 -0
  75. clue_api-1.0.0/clue/remote/datatypes/queues/__init__.py +0 -0
  76. clue_api-1.0.0/clue/remote/datatypes/queues/comms.py +62 -0
  77. clue_api-1.0.0/clue/remote/datatypes/set.py +96 -0
  78. clue_api-1.0.0/clue/remote/datatypes/user_quota_tracker.py +54 -0
  79. clue_api-1.0.0/clue/security/__init__.py +211 -0
  80. clue_api-1.0.0/clue/security/obo.py +87 -0
  81. clue_api-1.0.0/clue/security/utils.py +34 -0
  82. clue_api-1.0.0/clue/services/action_service.py +186 -0
  83. clue_api-1.0.0/clue/services/auth_service.py +348 -0
  84. clue_api-1.0.0/clue/services/config_service.py +38 -0
  85. clue_api-1.0.0/clue/services/fetcher_service.py +203 -0
  86. clue_api-1.0.0/clue/services/jwt_service.py +233 -0
  87. clue_api-1.0.0/clue/services/lookup_service.py +786 -0
  88. clue_api-1.0.0/clue/services/type_service.py +165 -0
  89. clue_api-1.0.0/clue/services/user_service.py +152 -0
  90. clue_api-1.0.0/pyproject.toml +286 -0
clue_api-1.0.0/LICENSE ADDED
@@ -0,0 +1,11 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2020 Crown Copyright, Government of Canada (Canadian Centre for Cyber Security / Communications Security Establishment)
4
+
5
+ Copyright title to all 3rd party software distributed with Clue is held by the respective copyright holders as noted in those files. Users are asked to read the 3rd Party Licenses referenced with those assets.
6
+
7
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
8
+
9
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
10
+
11
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,110 @@
1
+ Metadata-Version: 2.4
2
+ Name: clue-api
3
+ Version: 1.0.0
4
+ Summary: Clue distributed enrichment service
5
+ License: MIT
6
+ License-File: LICENSE
7
+ Keywords: clue,distributed,enrichment,gc,canada,cse-cst,cse,cst,cyber,cccs
8
+ Author: Canadian Centre for Cyber Security
9
+ Author-email: contact@cyber.gc.ca
10
+ Requires-Python: >=3.12,<4.0
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Classifier: Topic :: Software Development :: Libraries
19
+ Provides-Extra: server
20
+ Requires-Dist: PyYAML (>=6.0.1,<7.0.0) ; extra == "server"
21
+ Requires-Dist: Werkzeug (>=3.0.2,<4.0.0) ; extra == "server"
22
+ Requires-Dist: apscheduler (>=3.10.4,<4.0.0) ; extra == "server"
23
+ Requires-Dist: authlib (<1.0.0) ; extra == "server"
24
+ Requires-Dist: bcrypt (>=4.1.2,<5.0.0) ; extra == "server"
25
+ Requires-Dist: beautifulsoup4 (>=4.13.3,<5.0.0)
26
+ Requires-Dist: cart (>=1.2.3,<2.0.0)
27
+ Requires-Dist: elastic-apm (>=6.22.0,<7.0.0)
28
+ Requires-Dist: flasgger (>=0.9.7.1,<0.10.0.0) ; extra == "server"
29
+ Requires-Dist: flask (<3.0.0)
30
+ Requires-Dist: flask-caching (>=2.1.0,<3.0.0)
31
+ Requires-Dist: flask-cors (>=4.0.1,<5.0.0) ; extra == "server"
32
+ Requires-Dist: gevent (>=24.2.1,<25.0.0)
33
+ Requires-Dist: geventhttpclient (>=2.3.1,<3.0.0)
34
+ Requires-Dist: gunicorn (>=22,<24)
35
+ Requires-Dist: imgkit (>=1.2.3,<2.0.0)
36
+ Requires-Dist: passlib (>=1.7.4,<2.0.0) ; extra == "server"
37
+ Requires-Dist: pillow (>=11.1.0,<12.0.0)
38
+ Requires-Dist: prometheus-client (>=0.20.0,<0.21.0) ; extra == "server"
39
+ Requires-Dist: pydantic (>=2.7.1,<3.0.0)
40
+ Requires-Dist: pydantic-settings[yaml] (>=2.3.4,<3.0.0)
41
+ Requires-Dist: pyjwt (>=2.8.0,<3.0.0) ; extra == "server"
42
+ Requires-Dist: pyroute2 (>=0.7.12,<0.8.0) ; extra == "server"
43
+ Requires-Dist: python-baseconv (>=1.2.2,<2.0.0) ; extra == "server"
44
+ Requires-Dist: pytz (>=2024.1,<2025.0) ; extra == "server"
45
+ Requires-Dist: redis (>=5.0.3,<6.0.0)
46
+ Requires-Dist: requests (>=2.32.5,<3.0.0)
47
+ Requires-Dist: setuptools (<79.0.0)
48
+ Requires-Dist: trino (>=0.336.0,<0.337.0)
49
+ Project-URL: Documentation, https://github.com/CybercentreCanada/clue
50
+ Project-URL: Homepage, https://github.com/CybercentreCanada/clue
51
+ Project-URL: Repository, https://github.com/CybercentreCanada/clue
52
+ Description-Content-Type: text/markdown
53
+
54
+ # Clue
55
+
56
+ To start the API for clue, check to ensure that:
57
+
58
+ 1. Docker is composed up through `dev/docker-compose.yml`
59
+ 1. Note that you may need to set up uchimera container connections if you have not tyet done so:
60
+ 2. `az login && az acr login -n uchimera`
61
+ 3. If you do not have permission, reach out to APA2B.
62
+ 2. `cd clue/api`
63
+ 3. Run `poetry install` within the clue/api folder to install all dependencies
64
+ 4. You may need to run `poetry install --with test,dev,types,plugins --all-extras`
65
+ 5. Run `sudo mkdir -p /var/log/clue/`
66
+ 6. Run `sudo mkdir -p /etc/clue/conf/`
67
+ 7. Run `sudo chmod a+rw /var/log/clue/`
68
+ 8. Run `sudo chmod a+rw /etc/clue/conf/`
69
+ 9. Run `cp build_scripts/classification.yml /etc/clue/conf/classification.yml`
70
+ 10. Run `cp test/unit/config.yml /etc/clue/conf/config.yml`
71
+ 11. To start server: `poetry run server`
72
+
73
+ To start Enrichment Testing:
74
+
75
+ * In order to have the local server connect to the UI the servers need to be ran manually
76
+ * Please ensure that ```pwd``` is clue/api
77
+ * May need to add ```poetry run``` before each command
78
+
79
+ 1. ```flask --app test.utils.test_server run --no-reload --port 5008```
80
+ 2. ```flask --app test.utils.bad_server run --no-reload --port 5009```
81
+ 3. ```flask --app test.utils.slow_server run --no-reload --port 5010```
82
+ 4. ```flask --app test.utils.telemetry_server run --no-reload --port 5011```
83
+
84
+ Troubleshooting:
85
+
86
+ 1. If there are issues with these steps please check the build system for poetry installation steps
87
+ 2. The scripts will show all necessary directories that need to be made in order for classfication to work
88
+
89
+ ## Contributing
90
+
91
+ See [CONTRIBUTING.md](documentation/CONTRIBUTING.md) for more information
92
+
93
+ ## FAQ
94
+
95
+ ### I'm getting permissions issues on `/var/log/clue` or `/etc/clue/conf`?
96
+
97
+ Run `sudo chmod a+rw /var/log/clue/` and `sudo chmod a+rw /etc/clue/conf/`.
98
+
99
+ ### How can I add dependencies for my plugin?
100
+
101
+ See [this section](documentation/CONTRIBUTING.md#external-dependencies) of CONTRIBUTING.md.
102
+
103
+ ### Email rendering does not seem to be working?
104
+
105
+ You must install `wkhtmltopdf`, both locally for development and in your Dockerfile:
106
+
107
+ ```bash
108
+ sudo apt install wkhtmltopdf
109
+ ```
110
+
@@ -0,0 +1,56 @@
1
+ # Clue
2
+
3
+ To start the API for clue, check to ensure that:
4
+
5
+ 1. Docker is composed up through `dev/docker-compose.yml`
6
+ 1. Note that you may need to set up uchimera container connections if you have not tyet done so:
7
+ 2. `az login && az acr login -n uchimera`
8
+ 3. If you do not have permission, reach out to APA2B.
9
+ 2. `cd clue/api`
10
+ 3. Run `poetry install` within the clue/api folder to install all dependencies
11
+ 4. You may need to run `poetry install --with test,dev,types,plugins --all-extras`
12
+ 5. Run `sudo mkdir -p /var/log/clue/`
13
+ 6. Run `sudo mkdir -p /etc/clue/conf/`
14
+ 7. Run `sudo chmod a+rw /var/log/clue/`
15
+ 8. Run `sudo chmod a+rw /etc/clue/conf/`
16
+ 9. Run `cp build_scripts/classification.yml /etc/clue/conf/classification.yml`
17
+ 10. Run `cp test/unit/config.yml /etc/clue/conf/config.yml`
18
+ 11. To start server: `poetry run server`
19
+
20
+ To start Enrichment Testing:
21
+
22
+ * In order to have the local server connect to the UI the servers need to be ran manually
23
+ * Please ensure that ```pwd``` is clue/api
24
+ * May need to add ```poetry run``` before each command
25
+
26
+ 1. ```flask --app test.utils.test_server run --no-reload --port 5008```
27
+ 2. ```flask --app test.utils.bad_server run --no-reload --port 5009```
28
+ 3. ```flask --app test.utils.slow_server run --no-reload --port 5010```
29
+ 4. ```flask --app test.utils.telemetry_server run --no-reload --port 5011```
30
+
31
+ Troubleshooting:
32
+
33
+ 1. If there are issues with these steps please check the build system for poetry installation steps
34
+ 2. The scripts will show all necessary directories that need to be made in order for classfication to work
35
+
36
+ ## Contributing
37
+
38
+ See [CONTRIBUTING.md](documentation/CONTRIBUTING.md) for more information
39
+
40
+ ## FAQ
41
+
42
+ ### I'm getting permissions issues on `/var/log/clue` or `/etc/clue/conf`?
43
+
44
+ Run `sudo chmod a+rw /var/log/clue/` and `sudo chmod a+rw /etc/clue/conf/`.
45
+
46
+ ### How can I add dependencies for my plugin?
47
+
48
+ See [this section](documentation/CONTRIBUTING.md#external-dependencies) of CONTRIBUTING.md.
49
+
50
+ ### Email rendering does not seem to be working?
51
+
52
+ You must install `wkhtmltopdf`, both locally for development and in your Dockerfile:
53
+
54
+ ```bash
55
+ sudo apt install wkhtmltopdf
56
+ ```
@@ -0,0 +1,21 @@
1
+ # Add any directories, files, or patterns you don't want to be tracked by version control
2
+
3
+ # IDE files
4
+ .pydevproject
5
+ .python-version
6
+ .idea
7
+
8
+ # Testing artifacts
9
+ .pytest_cache
10
+ htmlcov
11
+ .coverage
12
+
13
+ # OS Files
14
+ ehthumbs.db
15
+ Thumbs.db
16
+
17
+ # Build artifacts
18
+ build/
19
+ dist/
20
+ *.py[cod]
21
+ *.egg-info
File without changes
@@ -0,0 +1,211 @@
1
+ from sys import exc_info
2
+ from traceback import format_tb
3
+ from typing import Any, Union
4
+
5
+ from flask import Blueprint, Response, make_response, request
6
+ from prometheus_client import Counter
7
+
8
+ from clue.common.forge import APP_NAME
9
+ from clue.common.logging import get_logger, log_with_traceback
10
+ from clue.common.str_utils import safe_str
11
+ from clue.models.network import ClueResponse
12
+
13
+ API_PREFIX = "/api"
14
+ RAW_API_COUNTER = Counter(
15
+ f"{APP_NAME.replace('-', '_')}_http_requests_total", # type: ignore[union-attr]
16
+ "HTTP Requests broken down by method, path, and status",
17
+ ["method", "path", "status"],
18
+ )
19
+
20
+ logger = get_logger(__file__)
21
+
22
+
23
+ def make_subapi_blueprint(name, api_version=1):
24
+ """Create a flask Blueprint for a subapi in a standard way."""
25
+ return Blueprint(name, name, url_prefix="/".join([API_PREFIX, f"v{api_version}", name]))
26
+
27
+
28
+ def _make_api_response(
29
+ data: Any, err: Union[str, Exception] = "", warnings: list[str] = [], status_code: int = 200, cookies: Any = None
30
+ ) -> Response:
31
+ if isinstance(err, Exception): # pragma: no cover
32
+ trace = exc_info()[2]
33
+ err = "".join(["\n"] + format_tb(trace) + ["%s: %s\n" % (err.__class__.__name__, str(err))]).rstrip("\n")
34
+ log_with_traceback(trace, "Exception", is_exception=True)
35
+
36
+ resp = make_response(
37
+ ClueResponse(response=data, error_message=err, warning=warnings, status_code=status_code).model_dump(
38
+ mode="json", by_alias=True, exclude_none=True
39
+ ),
40
+ status_code,
41
+ )
42
+
43
+ resp.headers["Content-Type"] = "application/json"
44
+
45
+ if isinstance(cookies, dict):
46
+ for k, v in cookies.items():
47
+ resp.set_cookie(k, v)
48
+
49
+ RAW_API_COUNTER.labels(request.method, str(request.url_rule), status_code).inc()
50
+ logger.info("%s %s - %s", request.method, request.path, status_code)
51
+
52
+ return resp
53
+
54
+
55
+ # Some helper functions for make_api_response
56
+
57
+ DEFAULT_DATA = {True: {"success": True}, False: {"success": False}}
58
+
59
+
60
+ def ok(data=DEFAULT_DATA[True], cookies=None):
61
+ """Returns response with status code 200"""
62
+ return _make_api_response(data, status_code=200, cookies=cookies)
63
+
64
+
65
+ def created(data=DEFAULT_DATA[True], warnings=[], cookies=None):
66
+ """Returns response with status code 201"""
67
+ return _make_api_response(data, warnings=warnings, status_code=201, cookies=cookies)
68
+
69
+
70
+ def accepted(data=DEFAULT_DATA[True], cookies=None):
71
+ """Returns response with status code 202"""
72
+ return _make_api_response(data, status_code=202, cookies=cookies)
73
+
74
+
75
+ def no_content(data=None, cookies=None):
76
+ """Returns response with status code 204"""
77
+ return _make_api_response(data or DEFAULT_DATA[True], status_code=204, cookies=cookies)
78
+
79
+
80
+ def not_modified(data=DEFAULT_DATA[True], cookies=None):
81
+ """Returns response with status code 304"""
82
+ return _make_api_response(data, status_code=304, cookies=cookies)
83
+
84
+
85
+ def bad_request(data=DEFAULT_DATA[False], err="", cookies=None, warnings=[]):
86
+ """Returns response with status code ies"""
87
+ return _make_api_response(data, err, status_code=400, cookies=cookies, warnings=warnings)
88
+
89
+
90
+ def unauthorized(data=DEFAULT_DATA[False], err="", cookies=None):
91
+ """Returns response with status code 401"""
92
+ return _make_api_response(data, err, status_code=401, cookies=cookies)
93
+
94
+
95
+ def forbidden(data=DEFAULT_DATA[False], err="", cookies=None):
96
+ """Returns response with status code 403"""
97
+ return _make_api_response(data, err, status_code=403, cookies=cookies)
98
+
99
+
100
+ def not_found(data=DEFAULT_DATA[False], err="", cookies=None):
101
+ """Returns response with status code 404"""
102
+ return _make_api_response(data, err, status_code=404, cookies=cookies)
103
+
104
+
105
+ def conflict(data=DEFAULT_DATA[False], err="", cookies=None):
106
+ """Returns response with status code 409"""
107
+ return _make_api_response(data, err, status_code=409, cookies=cookies)
108
+
109
+
110
+ def precondition_failed(data=DEFAULT_DATA[False], err="", cookies=None):
111
+ """Returns response with status code 412"""
112
+ return _make_api_response(data, err, status_code=412, cookies=cookies)
113
+
114
+
115
+ def teapot(data={**DEFAULT_DATA[False], "teapot": True}, err="", cookies=None):
116
+ """Returns response with status code 418"""
117
+ return _make_api_response(data, err, status_code=418, cookies=cookies)
118
+
119
+
120
+ def too_many_requests(data=DEFAULT_DATA[False], err="", cookies=None):
121
+ """Returns response with status code 429"""
122
+ return _make_api_response(data, err, status_code=429, cookies=cookies)
123
+
124
+
125
+ def internal_error(
126
+ data={**DEFAULT_DATA[False]},
127
+ err="Something went wrong. Contact an administrator.",
128
+ cookies=None,
129
+ ):
130
+ """Returns response with status code 500"""
131
+ return _make_api_response(data, err, status_code=500, cookies=cookies)
132
+
133
+
134
+ def not_implemented(
135
+ data={**DEFAULT_DATA[False]},
136
+ err="Something went wrong. Contact an administrator.",
137
+ cookies=None,
138
+ ):
139
+ """Returns response with status code 501"""
140
+ return _make_api_response(data, err, status_code=501, cookies=cookies)
141
+
142
+
143
+ def bad_gateway(
144
+ data={**DEFAULT_DATA[False]},
145
+ err="Something went wrong. Contact an administrator.",
146
+ cookies=None,
147
+ ):
148
+ """Returns response with status code 502"""
149
+ return _make_api_response(data, err, status_code=502, cookies=cookies)
150
+
151
+
152
+ def service_unavailable(
153
+ data={**DEFAULT_DATA[False]},
154
+ err="Something went wrong. Contact an administrator.",
155
+ cookies=None,
156
+ ):
157
+ """Returns response with status code 503"""
158
+ return _make_api_response(data, err, status_code=503, cookies=cookies)
159
+
160
+
161
+ def make_file_response(data, name, size, status_code=200, content_type="application/octet-stream"):
162
+ """Returns file response with arbitrary status code"""
163
+ response = make_response(data, status_code)
164
+ response.headers["Content-Type"] = content_type
165
+ response.headers["Content-Length"] = size
166
+ response.headers["Content-Disposition"] = 'attachment; filename="%s"' % safe_str(name)
167
+ return response
168
+
169
+
170
+ def stream_file_response(reader, name, size, status_code=200):
171
+ """Returns stream response with arbitrary status code"""
172
+ chunk_size = 65535
173
+
174
+ def generate():
175
+ reader.seek(0)
176
+ while True:
177
+ data = reader.read(chunk_size)
178
+ if not data:
179
+ break
180
+ yield data
181
+ reader.close()
182
+
183
+ headers = {
184
+ "Content-Type": "application/octet-stream",
185
+ "Content-Length": size,
186
+ "Content-Disposition": 'attachment; filename="%s"' % safe_str(name),
187
+ }
188
+ return Response(generate(), status=status_code, headers=headers)
189
+
190
+
191
+ def make_binary_response(data, size, status_code=200):
192
+ """Returns binary response with arbitrary status code"""
193
+ response = make_response(data, status_code)
194
+ response.headers["Content-Type"] = "application/octet-stream"
195
+ response.headers["Content-Length"] = size
196
+ return response
197
+
198
+
199
+ def stream_binary_response(reader, status_code=200):
200
+ """Returns streamed binary response with arbitrary status code"""
201
+ chunk_size = 4096
202
+
203
+ def generate():
204
+ reader.seek(0)
205
+ while True:
206
+ data = reader.read(chunk_size)
207
+ if not data:
208
+ break
209
+ yield data
210
+
211
+ return Response(generate(), status=status_code, mimetype="application/octet-stream")
@@ -0,0 +1,99 @@
1
+ from flask import Blueprint, current_app, request
2
+
3
+ from clue.api import ok
4
+ from clue.common.logging import get_logger
5
+ from clue.security import api_login
6
+
7
+ logger = get_logger(__file__)
8
+
9
+ API_PREFIX = "/api"
10
+ api = Blueprint("api", __name__, url_prefix=API_PREFIX)
11
+
12
+ XSRF_ENABLED = True
13
+
14
+
15
+ #####################################
16
+ # API list API (API inception)
17
+ @api.route("/")
18
+ @api_login(audit=False)
19
+ def api_version_list(**_):
20
+ """List all available API versions.
21
+
22
+ Variables:
23
+ None
24
+
25
+ Arguments:
26
+ None
27
+
28
+ Data Block:
29
+ None
30
+
31
+ Result example:
32
+ ["v1", "v2", "v3"] #List of API versions available
33
+ """
34
+ api_list = []
35
+ for rule in current_app.url_map.iter_rules():
36
+ if rule.rule.startswith("/api/"):
37
+ version = rule.rule[5:].split("/", 1)[0]
38
+ if version not in api_list and version != "":
39
+ # noinspection PyBroadException
40
+ try:
41
+ int(version[1:])
42
+ except ValueError:
43
+ continue
44
+ api_list.append(version)
45
+
46
+ return ok(api_list)
47
+
48
+
49
+ @api.route("/site_map/")
50
+ @api_login(audit=False)
51
+ def site_map(**_):
52
+ """Check if all pages have been protected by a login decorator
53
+
54
+ Variables:
55
+ None
56
+
57
+ Arguments:
58
+ unsafe_only => Only show unsafe pages
59
+
60
+ Data Block:
61
+ None
62
+
63
+ Result example:
64
+ [ #List of pages dictionary containing...
65
+ {"function": views.default, #Function name
66
+ "url": "/", #Url to page
67
+ "protected": true, #Is function login protected
68
+ "methods": ["GET"]}, #Methods allowed to access the page
69
+ ]
70
+ """
71
+ pages = []
72
+ for rule in current_app.url_map.iter_rules():
73
+ func = current_app.view_functions[rule.endpoint]
74
+ methods = []
75
+ if rule.methods:
76
+ for item in rule.methods:
77
+ if item != "OPTIONS" and item != "HEAD":
78
+ methods.append(item)
79
+ protected = func.__dict__.get("protected", False)
80
+ audit = func.__dict__.get("audit", False)
81
+ if "/api/v1/" in rule.rule:
82
+ prefix = "api.v1."
83
+ else:
84
+ prefix = ""
85
+
86
+ if "unsafe_only" in request.args and protected:
87
+ continue
88
+
89
+ pages.append(
90
+ {
91
+ "function": f"{prefix}{rule.endpoint.replace('apiv1.', '')}",
92
+ "url": rule.rule,
93
+ "methods": methods,
94
+ "protected": protected,
95
+ "audit": audit,
96
+ }
97
+ )
98
+
99
+ return ok(sorted(pages, key=lambda i: i["url"]))
@@ -0,0 +1,82 @@
1
+ from textwrap import dedent
2
+
3
+ from flask import Blueprint, current_app, request
4
+
5
+ from clue.api import ok
6
+ from clue.api.base import api_login
7
+
8
+ API_PREFIX = "/api/v1"
9
+ apiv1 = Blueprint("apiv1", __name__, url_prefix=API_PREFIX)
10
+ apiv1._doc = "Api Documentation - Verison 1" # type: ignore[attr-defined]
11
+
12
+
13
+ #####################################
14
+ # API DOCUMENTATION
15
+ # noinspection PyProtectedMember,PyBroadException
16
+ @apiv1.route("/")
17
+ @api_login(audit=False)
18
+ def get_api_documentation(**_):
19
+ """Full API doc. Loop through all registered API paths and display their documentation.
20
+
21
+ Returns a list of API definition.
22
+
23
+ Variables:
24
+ None
25
+
26
+ Arguments:
27
+ None
28
+
29
+ Result Example:
30
+ [
31
+ {'name': "Api Doc", # Name of the api
32
+ 'path': "/api/path/<variable>/", # API path
33
+ 'methods': ["GET", "POST"], # Allowed HTTP methods
34
+ 'description': "API doc.", # API documentation
35
+ 'id': "api_doc", # Unique ID for the API
36
+ 'function': "apiv1.api_doc", # Function called in the code
37
+ 'protected': False, # Does the API require login?
38
+ 'complete' : True}, # Is the API stable?
39
+ ...]
40
+ """
41
+ api_blueprints = {}
42
+ api_list = []
43
+ for rule in current_app.url_map.iter_rules():
44
+ if rule.rule.startswith(request.path):
45
+ methods = [item for item in (rule.methods or []) if item != "OPTIONS" and item != "HEAD"]
46
+
47
+ func = current_app.view_functions[rule.endpoint]
48
+ doc_string = func.__doc__
49
+ func_title = " ".join([x.capitalize() for x in rule.endpoint[rule.endpoint.rindex(".") + 1 :].split("_")])
50
+ blueprint = rule.endpoint[: rule.endpoint.rindex(".")]
51
+ if blueprint == "apiv1":
52
+ blueprint = "documentation"
53
+
54
+ if blueprint not in api_blueprints:
55
+ try:
56
+ doc = current_app.blueprints[rule.endpoint[: rule.endpoint.rindex(".")]]._doc # type: ignore[attr-defined]
57
+ except Exception:
58
+ doc = ""
59
+
60
+ api_blueprints[blueprint] = doc
61
+
62
+ if doc_string:
63
+ description = dedent(doc_string)
64
+ else:
65
+ description = "[INCOMPLETE]\n\nTHIS API HAS NOT BEEN DOCUMENTED YET!"
66
+
67
+ api_id = rule.endpoint.replace("apiv1.", "").replace(".", "_")
68
+
69
+ api_list.append(
70
+ {
71
+ "protected": func.__dict__.get("protected", False),
72
+ "name": func_title,
73
+ "id": api_id,
74
+ "function": f"api.v1.{rule.endpoint}",
75
+ "path": rule.rule,
76
+ "methods": methods,
77
+ "description": description,
78
+ "complete": "[INCOMPLETE]" not in description,
79
+ }
80
+ )
81
+
82
+ return ok({"apis": api_list, "blueprints": api_blueprints})
@@ -0,0 +1,92 @@
1
+ """Enrichment Actions
2
+
3
+ List and execute actions
4
+
5
+ * Provide endpoints to list valid actions exposed by plugins.
6
+ * Provide endpoints to execute these actions.
7
+ """
8
+
9
+ from flask_cors import CORS
10
+
11
+ from clue.api import internal_error, make_subapi_blueprint, not_found, ok
12
+ from clue.common.exceptions import ClueException, NotFoundException
13
+ from clue.common.logging import get_logger
14
+ from clue.common.swagger import generate_swagger_docs
15
+ from clue.config import config
16
+ from clue.models.actions import Action, ActionResult
17
+ from clue.security import api_login
18
+ from clue.services import action_service
19
+
20
+ logger = get_logger(__file__)
21
+
22
+
23
+ SUB_API = "actions"
24
+ actions_api = make_subapi_blueprint(SUB_API, api_version=1)
25
+ actions_api._doc = "Run actions on data through configured external data sources/systems."
26
+
27
+ CORS(actions_api, origins=config.ui.cors_origins, supports_credentials=True)
28
+
29
+
30
+ @generate_swagger_docs(responses={200: "A list of types and their classification"})
31
+ @actions_api.route("/", methods=["GET"])
32
+ @api_login()
33
+ def get_actions(**kwargs) -> dict[str, Action]:
34
+ """Return the supported actions of each external service.
35
+
36
+ Variables:
37
+ None
38
+
39
+ Arguments:
40
+ None
41
+
42
+ Result Example:
43
+ { # A dictionary of sources with their supported actions.
44
+ <source_id>.<action_id>: {
45
+ "id": "",
46
+ "name": "",
47
+ "classification": "",
48
+ "summary": "",
49
+ "supported_types": "",
50
+ "params": {
51
+ <JSON schema>
52
+ }
53
+ },
54
+ ...,
55
+ }
56
+ """
57
+ return ok(action_service.get_plugins_supported_actions(kwargs["user"]))
58
+
59
+
60
+ @generate_swagger_docs(responses={200: "Successful lookup to selected plugins"})
61
+ @actions_api.route("/execute/<plugin_id>/<action_id>", methods=["POST"])
62
+ @api_login()
63
+ def execute_action(plugin_id: str, action_id: str, **kwargs) -> ActionResult:
64
+ """Search other services for additional information related to the provided data.
65
+
66
+ Variables:
67
+ plugin_id (str): the ID of the plugin who owns the action to execute
68
+ action_id (str): the ID of the action to execute
69
+
70
+ Arguments:
71
+ None
72
+
73
+ Data Block:
74
+ {
75
+ type: "ip",
76
+ value: "127.0.0.1",
77
+ ...
78
+ }
79
+
80
+ Result Example:
81
+ {
82
+ "outcome": "success | failure", # was this execution a success or failure?
83
+ "format": "link", # What format is the output in?
84
+ "output": "http://example.com" # The output of the action. Can be any data structure.
85
+ }
86
+ """
87
+ try:
88
+ return ok(action_service.execute_action(plugin_id, action_id, kwargs["user"]))
89
+ except NotFoundException as err:
90
+ return not_found(err=err.message)
91
+ except ClueException as err:
92
+ return internal_error(err=err.message)