clue-api 1.0.0.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clue/.gitignore +21 -0
- clue/__init__.py +0 -0
- clue/api/__init__.py +211 -0
- clue/api/base.py +99 -0
- clue/api/v1/__init__.py +82 -0
- clue/api/v1/actions.py +92 -0
- clue/api/v1/auth.py +243 -0
- clue/api/v1/configs.py +83 -0
- clue/api/v1/fetchers.py +94 -0
- clue/api/v1/lookup.py +221 -0
- clue/api/v1/registration.py +109 -0
- clue/api/v1/static.py +94 -0
- clue/app.py +166 -0
- clue/cache/__init__.py +129 -0
- clue/common/__init__.py +0 -0
- clue/common/classification.py +1006 -0
- clue/common/classification.yml +130 -0
- clue/common/dict_utils.py +130 -0
- clue/common/exceptions.py +199 -0
- clue/common/forge.py +152 -0
- clue/common/json_utils.py +10 -0
- clue/common/list_utils.py +11 -0
- clue/common/logging/__init__.py +291 -0
- clue/common/logging/audit.py +157 -0
- clue/common/logging/format.py +42 -0
- clue/common/regex.py +31 -0
- clue/common/str_utils.py +213 -0
- clue/common/swagger.py +139 -0
- clue/common/uid.py +47 -0
- clue/config.py +60 -0
- clue/constants/__init__.py +0 -0
- clue/constants/supported_types.py +38 -0
- clue/cronjobs/__init__.py +30 -0
- clue/cronjobs/plugins.py +32 -0
- clue/error.py +129 -0
- clue/gunicorn_config.py +29 -0
- clue/healthz.py +74 -0
- clue/helper/discover.py +53 -0
- clue/helper/headers.py +30 -0
- clue/helper/oauth.py +128 -0
- clue/models/__init__.py +0 -0
- clue/models/actions.py +243 -0
- clue/models/config.py +456 -0
- clue/models/fetchers.py +136 -0
- clue/models/graph.py +162 -0
- clue/models/model_list.py +52 -0
- clue/models/network.py +430 -0
- clue/models/results/__init__.py +34 -0
- clue/models/results/base.py +10 -0
- clue/models/results/graph.py +26 -0
- clue/models/results/image.py +22 -0
- clue/models/results/status.py +55 -0
- clue/models/results/validation.py +57 -0
- clue/models/selector.py +67 -0
- clue/models/utils.py +52 -0
- clue/models/validators.py +19 -0
- clue/patched.py +8 -0
- clue/plugin/__init__.py +1008 -0
- clue/plugin/helpers/__init__.py +0 -0
- clue/plugin/helpers/central_server.py +27 -0
- clue/plugin/helpers/email_render.py +228 -0
- clue/plugin/helpers/token.py +34 -0
- clue/plugin/helpers/trino.py +103 -0
- clue/plugin/interactive.py +270 -0
- clue/plugin/models.py +19 -0
- clue/plugin/utils.py +78 -0
- clue/remote/__init__.py +0 -0
- clue/remote/datatypes/__init__.py +130 -0
- clue/remote/datatypes/cache.py +62 -0
- clue/remote/datatypes/events.py +118 -0
- clue/remote/datatypes/hash.py +193 -0
- clue/remote/datatypes/queues/__init__.py +0 -0
- clue/remote/datatypes/queues/comms.py +62 -0
- clue/remote/datatypes/set.py +96 -0
- clue/remote/datatypes/user_quota_tracker.py +54 -0
- clue/security/__init__.py +211 -0
- clue/security/obo.py +95 -0
- clue/security/utils.py +34 -0
- clue/services/action_service.py +186 -0
- clue/services/auth_service.py +348 -0
- clue/services/config_service.py +38 -0
- clue/services/fetcher_service.py +203 -0
- clue/services/jwt_service.py +233 -0
- clue/services/lookup_service.py +786 -0
- clue/services/type_service.py +165 -0
- clue/services/user_service.py +152 -0
- clue_api-1.0.0.dev7.dist-info/METADATA +111 -0
- clue_api-1.0.0.dev7.dist-info/RECORD +91 -0
- clue_api-1.0.0.dev7.dist-info/WHEEL +4 -0
- clue_api-1.0.0.dev7.dist-info/entry_points.txt +8 -0
- clue_api-1.0.0.dev7.dist-info/licenses/LICENSE +11 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
import elasticapm
|
|
4
|
+
import requests
|
|
5
|
+
from flask import request
|
|
6
|
+
from requests import exceptions
|
|
7
|
+
|
|
8
|
+
from clue.common.logging import get_logger
|
|
9
|
+
from clue.config import CLASSIFICATION, DEBUG, cache, config
|
|
10
|
+
from clue.constants.supported_types import SUPPORTED_TYPES
|
|
11
|
+
from clue.helper.headers import generate_headers
|
|
12
|
+
from clue.models.config import ExternalSource
|
|
13
|
+
from clue.remote.datatypes.cache import RedisCache
|
|
14
|
+
from clue.services import auth_service
|
|
15
|
+
|
|
16
|
+
logger = get_logger(__file__)
|
|
17
|
+
|
|
18
|
+
# Either cache for one second in debug mode, or five minutes in production
|
|
19
|
+
CACHE_TIMEOUT: int = 1 if DEBUG else 5 * 60
|
|
20
|
+
CACHE = RedisCache(prefix="brl_types", ttl=CACHE_TIMEOUT)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_types_regular_expressions(user: dict[str, Any]):
|
|
24
|
+
"""Return the regular expression to detect the different types"""
|
|
25
|
+
access_token = request.headers.get("Authorization", type=str)
|
|
26
|
+
if access_token:
|
|
27
|
+
access_token = access_token.split(" ")[1]
|
|
28
|
+
|
|
29
|
+
all_types = all_supported_types(
|
|
30
|
+
user,
|
|
31
|
+
access_token=access_token,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
type_detection = {}
|
|
35
|
+
|
|
36
|
+
for source_types in all_types.values():
|
|
37
|
+
for data_type, classification in source_types.items():
|
|
38
|
+
# Validate if the user is allow to even see the source
|
|
39
|
+
if user and not CLASSIFICATION.is_accessible(user["classification"], classification):
|
|
40
|
+
continue
|
|
41
|
+
|
|
42
|
+
type_detection[data_type] = SUPPORTED_TYPES[data_type]
|
|
43
|
+
|
|
44
|
+
return type_detection
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@cache.memoize(timeout=CACHE_TIMEOUT)
|
|
48
|
+
def get_supported_types(source_url: str, access_token: str | None = None, obo_access_token: str | None = None):
|
|
49
|
+
"""Gets all supported types for the specified source.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
source_url (str): The url of the source.
|
|
53
|
+
access_token (str | None, optional): An access token giving access to the source. Defaults to None.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
Any: The supported types returned by the source.
|
|
57
|
+
"""
|
|
58
|
+
url = f"{source_url}{'' if source_url.endswith('/') else '/'}types/"
|
|
59
|
+
|
|
60
|
+
if result := CACHE.get(url):
|
|
61
|
+
logger.info("Cache hit for url %s", url)
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
logger.debug("Cache miss, polling plugin")
|
|
65
|
+
with elasticapm.capture_span(f"GET {url}", span_type="http"):
|
|
66
|
+
headers = generate_headers(obo_access_token or access_token, access_token if obo_access_token else None)
|
|
67
|
+
|
|
68
|
+
try:
|
|
69
|
+
rsp = requests.get(url, headers=headers, timeout=3.0)
|
|
70
|
+
except exceptions.ConnectionError:
|
|
71
|
+
# any errors are logged and no result is saved to local cache to enable retry on next query
|
|
72
|
+
logger.exception(f"Unable to connect: {url}")
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
status_code = rsp.status_code
|
|
76
|
+
if status_code != 200:
|
|
77
|
+
try:
|
|
78
|
+
err = rsp.json()["api_error_message"]
|
|
79
|
+
logger.error(f"Error ({rsp.status_code}) from upstream server: {status_code=}, {err=}")
|
|
80
|
+
return None
|
|
81
|
+
except requests.exceptions.JSONDecodeError:
|
|
82
|
+
logger.exception(
|
|
83
|
+
f"Parsing error in error ({rsp.status_code}) response - unknown format\n"
|
|
84
|
+
f"Raw response: {rsp.content}"
|
|
85
|
+
)
|
|
86
|
+
return None
|
|
87
|
+
except KeyError:
|
|
88
|
+
logger.exception(
|
|
89
|
+
f"Parsing error in error ({rsp.status_code}) response - 'api_error_message' is missing\n",
|
|
90
|
+
f"Full response: {rsp.json()}",
|
|
91
|
+
)
|
|
92
|
+
return None
|
|
93
|
+
except Exception:
|
|
94
|
+
content = rsp.content
|
|
95
|
+
if isinstance(content, (bytes, bytearray)):
|
|
96
|
+
content = content.decode()
|
|
97
|
+
logger.exception(f"{source_url} encountered an unknown error.\n" f"Full response: {content}")
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
try:
|
|
101
|
+
types_result = rsp.json()["api_response"]
|
|
102
|
+
logger.debug("Setting cache result for url %s", url)
|
|
103
|
+
CACHE.set(url, types_result)
|
|
104
|
+
return types_result
|
|
105
|
+
except requests.exceptions.JSONDecodeError:
|
|
106
|
+
logger.exception("Parsing error in OK response - unknown format\n" f"Raw response: {rsp.content}")
|
|
107
|
+
return None
|
|
108
|
+
except Exception:
|
|
109
|
+
logger.exception("External API did not return expected format:")
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def all_supported_types(user: dict[str, Any], access_token: str | None = None) -> dict[str, dict[str, str]]:
|
|
114
|
+
"""Gets supported types by all sources.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
access_token (str | None, optional): An access token giving access to the sources. Defaults to None.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
dict[str, dict[str, str]]: A dict of each source and their supported types.
|
|
121
|
+
"""
|
|
122
|
+
all_types = {}
|
|
123
|
+
|
|
124
|
+
for source in config.api.external_sources:
|
|
125
|
+
obo_access_token = None
|
|
126
|
+
if access_token:
|
|
127
|
+
obo_access_token, error = auth_service.check_obo(source, access_token, user["uname"])
|
|
128
|
+
|
|
129
|
+
if error:
|
|
130
|
+
logger.error("%s: %s", source.name, error)
|
|
131
|
+
|
|
132
|
+
supported_types = get_supported_types(source.url, access_token=access_token, obo_access_token=obo_access_token)
|
|
133
|
+
if supported_types is not None:
|
|
134
|
+
all_types[source.name] = {k: v for k, v in supported_types.items() if k in SUPPORTED_TYPES}
|
|
135
|
+
|
|
136
|
+
return all_types
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def get_plugins_supported_types(user: dict[str, Any]):
|
|
140
|
+
"""Return the supported type names of each external service, filtered to what the user has access to."""
|
|
141
|
+
configured_sources: list[ExternalSource] = getattr(config.api, "external_sources", [])
|
|
142
|
+
available_types: dict[str, list[str]] = {}
|
|
143
|
+
|
|
144
|
+
access_token = request.headers.get("Authorization", type=str)
|
|
145
|
+
if access_token:
|
|
146
|
+
access_token = access_token.split(" ")[1]
|
|
147
|
+
|
|
148
|
+
all_types = all_supported_types(user, access_token=access_token)
|
|
149
|
+
|
|
150
|
+
logger.info("Fetching sources for classification %s", user["classification"])
|
|
151
|
+
|
|
152
|
+
for source in configured_sources:
|
|
153
|
+
# Validate if the user is allow to even see the source
|
|
154
|
+
if user and not CLASSIFICATION.is_accessible(user["classification"], source.classification):
|
|
155
|
+
logger.info("Not including source %s at classification %s", source.name, user["classification"])
|
|
156
|
+
continue
|
|
157
|
+
|
|
158
|
+
# user can view source, now filter types user cannot see
|
|
159
|
+
available_types[source.name] = [
|
|
160
|
+
tname
|
|
161
|
+
for tname, classification in all_types.get(source.name, {}).items()
|
|
162
|
+
if user and CLASSIFICATION.is_accessible(user["classification"], classification)
|
|
163
|
+
]
|
|
164
|
+
|
|
165
|
+
return available_types
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
import elasticapm
|
|
4
|
+
from flask import current_app
|
|
5
|
+
|
|
6
|
+
from clue.common.exceptions import (
|
|
7
|
+
AccessDeniedException,
|
|
8
|
+
ClueValueError,
|
|
9
|
+
InvalidDataException,
|
|
10
|
+
)
|
|
11
|
+
from clue.common.logging import get_logger
|
|
12
|
+
from clue.config import CLASSIFICATION, config, get_redis
|
|
13
|
+
from clue.helper.oauth import parse_profile
|
|
14
|
+
from clue.models.config import ExternalSource
|
|
15
|
+
from clue.remote.datatypes.user_quota_tracker import UserQuotaTracker
|
|
16
|
+
|
|
17
|
+
logger = get_logger(__file__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@elasticapm.capture_span(span_type="authentication")
|
|
21
|
+
def parse_user_data(
|
|
22
|
+
data: dict,
|
|
23
|
+
oauth_provider: str,
|
|
24
|
+
) -> dict[str, Any]:
|
|
25
|
+
"""Convert a JSON Web Token into a Clue User
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
data (dict): The JWT to parse
|
|
29
|
+
oauth_provider (str): The provider of the JWT
|
|
30
|
+
skip_setup (bool, optional): Skip the extra setup steps we run at login, for performance reasons.
|
|
31
|
+
Defaults to True.
|
|
32
|
+
access_token (str, optional): The access token to use when fetching the user's avatar. Defaults to None.
|
|
33
|
+
|
|
34
|
+
Raises:
|
|
35
|
+
InvalidDataException: Some required data was missing.
|
|
36
|
+
AccessDeniedException: The user is not permitted to access the application, or user auto-creation is disabled
|
|
37
|
+
and the user doesn't exist in the database.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
User: The parsed User ODM
|
|
41
|
+
"""
|
|
42
|
+
if not data or not oauth_provider:
|
|
43
|
+
raise InvalidDataException("Both the JWT and OAuth provider must be supplied")
|
|
44
|
+
|
|
45
|
+
oauth = current_app.extensions.get("authlib.integrations.flask_client")
|
|
46
|
+
if not oauth:
|
|
47
|
+
logger.critical("Authlib integration missing!")
|
|
48
|
+
raise ClueValueError()
|
|
49
|
+
provider = oauth.create_client(oauth_provider)
|
|
50
|
+
|
|
51
|
+
if "id_token" in data:
|
|
52
|
+
data = provider.parse_id_token(data)
|
|
53
|
+
|
|
54
|
+
oauth_provider_config = config.auth.oauth.providers[oauth_provider]
|
|
55
|
+
|
|
56
|
+
if not data:
|
|
57
|
+
raise AccessDeniedException("Not user data contained in the token")
|
|
58
|
+
|
|
59
|
+
user_data = parse_profile(data, oauth_provider_config)
|
|
60
|
+
if len(oauth_provider_config.required_groups) > 0:
|
|
61
|
+
required_groups = set(oauth_provider_config.required_groups)
|
|
62
|
+
if len(required_groups) != len(required_groups & set(user_data["groups"])):
|
|
63
|
+
logger.warning(
|
|
64
|
+
f"User {user_data['uname']} is missing groups from their JWT:"
|
|
65
|
+
f" {', '.join(required_groups - (required_groups & set(user_data['groups'])))}"
|
|
66
|
+
)
|
|
67
|
+
raise AccessDeniedException("This user is not allowed access to the system")
|
|
68
|
+
|
|
69
|
+
has_access = user_data.pop("access", False)
|
|
70
|
+
if has_access and user_data["email"] is not None:
|
|
71
|
+
user_data["uname"]
|
|
72
|
+
|
|
73
|
+
# Add add dynamic classification group
|
|
74
|
+
get_dynamic_classification(user_data, oauth_provider)
|
|
75
|
+
else:
|
|
76
|
+
raise AccessDeniedException("This user is not allowed access to the system")
|
|
77
|
+
|
|
78
|
+
return user_data
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def get_dynamic_classification(user_data: dict[str, Any], oauth_provider: str):
|
|
82
|
+
"""Get the classification of the user
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
current_c12n (str): The current classification of the user
|
|
86
|
+
email (str): The user's email
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
str: The classification
|
|
90
|
+
"""
|
|
91
|
+
classification_map = config.auth.oauth.providers[oauth_provider].classification_map
|
|
92
|
+
if len(user_data["groups"]) > 0 and classification_map:
|
|
93
|
+
for group in user_data["groups"]:
|
|
94
|
+
if group in classification_map:
|
|
95
|
+
if not CLASSIFICATION.is_valid(classification_map[group]):
|
|
96
|
+
logger.warning("Group %s has invalid classification mapping %s", group, classification_map[group])
|
|
97
|
+
continue
|
|
98
|
+
|
|
99
|
+
user_data["classification"] = CLASSIFICATION.max_classification(
|
|
100
|
+
user_data["classification"], classification_map[group]
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
QUOTA_TRACKERS: dict[str, UserQuotaTracker] = {}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def check_quota(source: ExternalSource, user: dict[str, Any]) -> str | None:
|
|
108
|
+
"Check that a user does not have too many concurrent requests to a given external service."
|
|
109
|
+
if not source.obo_target:
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
quota = config.api.obo_targets[source.obo_target].quota
|
|
113
|
+
|
|
114
|
+
if quota is None:
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
if source.obo_target not in QUOTA_TRACKERS:
|
|
118
|
+
QUOTA_TRACKERS[source.obo_target] = UserQuotaTracker(source.obo_target, timeout=60, redis=get_redis())
|
|
119
|
+
|
|
120
|
+
if QUOTA_TRACKERS[source.obo_target].begin(user["uname"], quota):
|
|
121
|
+
logger.debug(
|
|
122
|
+
"User %s is below quota of %s concurrent requests to source %s",
|
|
123
|
+
user["uname"],
|
|
124
|
+
quota,
|
|
125
|
+
source.obo_target,
|
|
126
|
+
)
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
logger.error(
|
|
130
|
+
"User %s has exceeded quota of %s concurrent requests to source %s",
|
|
131
|
+
user["uname"],
|
|
132
|
+
quota,
|
|
133
|
+
source.obo_target,
|
|
134
|
+
)
|
|
135
|
+
return (
|
|
136
|
+
f"You have too many simultaneous connections to external service {source.obo_target}. "
|
|
137
|
+
"Please use larger batches when enriching."
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def release_quota(source: ExternalSource, user: dict[str, Any]):
|
|
142
|
+
"Release the space claimed by a given request in the user's quota"
|
|
143
|
+
if not source.obo_target:
|
|
144
|
+
return
|
|
145
|
+
|
|
146
|
+
quota = config.api.obo_targets[source.obo_target].quota
|
|
147
|
+
|
|
148
|
+
if quota is None:
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
if source.obo_target in QUOTA_TRACKERS:
|
|
152
|
+
QUOTA_TRACKERS[source.obo_target].end(user["uname"])
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: clue-api
|
|
3
|
+
Version: 1.0.0.dev7
|
|
4
|
+
Summary: Clue distributed enrichment service
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: clue,distributed,enrichment,gc,canada,cse-cst,cse,cst,cyber,cccs
|
|
8
|
+
Author: Canadian Centre for Cyber Security
|
|
9
|
+
Author-email: contact@cyber.gc.ca
|
|
10
|
+
Requires-Python: >=3.12,<4.0
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Provides-Extra: server
|
|
20
|
+
Requires-Dist: PyYAML (>=6.0.1,<7.0.0) ; extra == "server"
|
|
21
|
+
Requires-Dist: Werkzeug (>=3.0.2,<4.0.0) ; extra == "server"
|
|
22
|
+
Requires-Dist: apscheduler (>=3.10.4,<4.0.0) ; extra == "server"
|
|
23
|
+
Requires-Dist: authlib (<1.0.0) ; extra == "server"
|
|
24
|
+
Requires-Dist: bcrypt (>=4.1.2,<5.0.0) ; extra == "server"
|
|
25
|
+
Requires-Dist: beautifulsoup4 (>=4.13.3,<5.0.0)
|
|
26
|
+
Requires-Dist: cart (>=1.2.3,<2.0.0)
|
|
27
|
+
Requires-Dist: elastic-apm (>=6.22.0,<7.0.0)
|
|
28
|
+
Requires-Dist: flasgger (>=0.9.7.1,<0.10.0.0) ; extra == "server"
|
|
29
|
+
Requires-Dist: flask (<3.0.0)
|
|
30
|
+
Requires-Dist: flask-caching (>=2.1.0,<3.0.0)
|
|
31
|
+
Requires-Dist: flask-cors (>=4.0.1,<5.0.0) ; extra == "server"
|
|
32
|
+
Requires-Dist: gevent (>=24.2.1,<25.0.0)
|
|
33
|
+
Requires-Dist: geventhttpclient (>=2.3.1,<3.0.0)
|
|
34
|
+
Requires-Dist: gunicorn (>=22.0.0,<23.0.0)
|
|
35
|
+
Requires-Dist: imgkit (>=1.2.3,<2.0.0)
|
|
36
|
+
Requires-Dist: netifaces (>=0.11.0,<0.12.0) ; extra == "server"
|
|
37
|
+
Requires-Dist: passlib (>=1.7.4,<2.0.0) ; extra == "server"
|
|
38
|
+
Requires-Dist: pillow (>=11.1.0,<12.0.0)
|
|
39
|
+
Requires-Dist: prometheus-client (>=0.20.0,<0.21.0) ; extra == "server"
|
|
40
|
+
Requires-Dist: pydantic (>=2.7.1,<3.0.0)
|
|
41
|
+
Requires-Dist: pydantic-settings[yaml] (>=2.3.4,<3.0.0)
|
|
42
|
+
Requires-Dist: pyjwt (>=2.8.0,<3.0.0) ; extra == "server"
|
|
43
|
+
Requires-Dist: pyroute2 (>=0.7.12,<0.8.0) ; extra == "server"
|
|
44
|
+
Requires-Dist: python-baseconv (>=1.2.2,<2.0.0) ; extra == "server"
|
|
45
|
+
Requires-Dist: pytz (>=2024.1,<2025.0) ; extra == "server"
|
|
46
|
+
Requires-Dist: redis (>=5.0.3,<6.0.0)
|
|
47
|
+
Requires-Dist: requests (>=2.32.5,<3.0.0)
|
|
48
|
+
Requires-Dist: setuptools (<78.0.0)
|
|
49
|
+
Requires-Dist: trino (>=0.336.0,<0.337.0)
|
|
50
|
+
Project-URL: Documentation, https://github.com/CybercentreCanada/clue
|
|
51
|
+
Project-URL: Homepage, https://github.com/CybercentreCanada/clue
|
|
52
|
+
Project-URL: Repository, https://github.com/CybercentreCanada/clue
|
|
53
|
+
Description-Content-Type: text/markdown
|
|
54
|
+
|
|
55
|
+
# Clue
|
|
56
|
+
|
|
57
|
+
To start the API for clue, check to ensure that:
|
|
58
|
+
|
|
59
|
+
1. Docker is composed up through `dev/docker-compose.yml`
|
|
60
|
+
1. Note that you may need to set up uchimera container connections if you have not tyet done so:
|
|
61
|
+
2. `az login && az acr login -n uchimera`
|
|
62
|
+
3. If you do not have permission, reach out to APA2B.
|
|
63
|
+
2. `cd clue/api`
|
|
64
|
+
3. Run `poetry install` within the clue/api folder to install all dependencies
|
|
65
|
+
4. You may need to run `poetry install --with test,dev,types,plugins --all-extras`
|
|
66
|
+
5. Run `sudo mkdir -p /var/log/clue/`
|
|
67
|
+
6. Run `sudo mkdir -p /etc/clue/conf/`
|
|
68
|
+
7. Run `sudo chmod a+rw /var/log/clue/`
|
|
69
|
+
8. Run `sudo chmod a+rw /etc/clue/conf/`
|
|
70
|
+
9. Run `cp build_scripts/classification.yml /etc/clue/conf/classification.yml`
|
|
71
|
+
10. Run `cp test/unit/config.yml /etc/clue/conf/config.yml`
|
|
72
|
+
11. To start server: `poetry run server`
|
|
73
|
+
|
|
74
|
+
To start Enrichment Testing:
|
|
75
|
+
|
|
76
|
+
* In order to have the local server connect to the UI the servers need to be ran manually
|
|
77
|
+
* Please ensure that ```pwd``` is clue/api
|
|
78
|
+
* May need to add ```poetry run``` before each command
|
|
79
|
+
|
|
80
|
+
1. ```flask --app test.utils.test_server run --no-reload --port 5008```
|
|
81
|
+
2. ```flask --app test.utils.bad_server run --no-reload --port 5009```
|
|
82
|
+
3. ```flask --app test.utils.slow_server run --no-reload --port 5010```
|
|
83
|
+
4. ```flask --app test.utils.telemetry_server run --no-reload --port 5011```
|
|
84
|
+
|
|
85
|
+
Troubleshooting:
|
|
86
|
+
|
|
87
|
+
1. If there are issues with these steps please check the build system for poetry installation steps
|
|
88
|
+
2. The scripts will show all necessary directories that need to be made in order for classfication to work
|
|
89
|
+
|
|
90
|
+
## Contributing
|
|
91
|
+
|
|
92
|
+
See [CONTRIBUTING.md](documentation/CONTRIBUTING.md) for more information
|
|
93
|
+
|
|
94
|
+
## FAQ
|
|
95
|
+
|
|
96
|
+
### I'm getting permissions issues on `/var/log/clue` or `/etc/clue/conf`?
|
|
97
|
+
|
|
98
|
+
Run `sudo chmod a+rw /var/log/clue/` and `sudo chmod a+rw /etc/clue/conf/`.
|
|
99
|
+
|
|
100
|
+
### How can I add dependencies for my plugin?
|
|
101
|
+
|
|
102
|
+
See [this section](documentation/CONTRIBUTING.md#external-dependencies) of CONTRIBUTING.md.
|
|
103
|
+
|
|
104
|
+
### Email rendering does not seem to be working?
|
|
105
|
+
|
|
106
|
+
You must install `wkhtmltopdf`, both locally for development and in your Dockerfile:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
sudo apt install wkhtmltopdf
|
|
110
|
+
```
|
|
111
|
+
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
clue/.gitignore,sha256=ovW5bRSExbACPwwUvWCvq3tVoerr8p41mIYOQgUqw2M,279
|
|
2
|
+
clue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
clue/api/__init__.py,sha256=T7x1BSRalguySE5V3M31oKdpC25FfOKUeuk1_nqXYno,7211
|
|
4
|
+
clue/api/base.py,sha256=2mdivAJ-XQRB7e7evd8vd-1BbsUH-NZxhmuEIblvg3Y,2634
|
|
5
|
+
clue/api/v1/__init__.py,sha256=iA9jRGp338CKl0ueogOUhb5ngojdRfRlgtKY4OvfAYM,2926
|
|
6
|
+
clue/api/v1/actions.py,sha256=_T0iBN_9WilodJjnX6t8-lBd8PyxMQtZxHdhEWPuilE,2735
|
|
7
|
+
clue/api/v1/auth.py,sha256=6IjujeiqCErdtVx2vEwZpDrrwsmMcw2x-Zz5xyI4ucM,9508
|
|
8
|
+
clue/api/v1/configs.py,sha256=lq6DGs839jzBYMsdBuLCGlSBnug3DaJ_VW6kW-WnFN0,2578
|
|
9
|
+
clue/api/v1/fetchers.py,sha256=NuDKxCf5ulVxUNBgGbeDfxRGPW7jAAuxQFSIFfUZnx8,3035
|
|
10
|
+
clue/api/v1/lookup.py,sha256=vC_m8dOHtDHAzOpSPPj52zWtmkjzgqaV9tOzuB5-w4k,8059
|
|
11
|
+
clue/api/v1/registration.py,sha256=kH5mPiRoXGiTV2IZGmyHA2KY9K0j1uGwpiPVluTC5HM,3365
|
|
12
|
+
clue/api/v1/static.py,sha256=hRW2sIQLtu6ZXoQtkNLCsquEbd9mOnAWEvwJliiG5g0,2795
|
|
13
|
+
clue/app.py,sha256=5QzcMHGb-uGKlMj6NW9_JmAEpq_4rqbK8PDgDt29XVg,5607
|
|
14
|
+
clue/cache/__init__.py,sha256=eFPje54qbR5gTWMZhs26w2m0h0KjvMCMIqpESkjVYaI,4383
|
|
15
|
+
clue/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
+
clue/common/classification.py,sha256=vnIU70rzvu-tVnZhXSKwwcedYbkRV4mqytmRZFBeSLY,41287
|
|
17
|
+
clue/common/classification.yml,sha256=DkxFpYkxdNQd-cAgaNS3cLMsn5g1om3uQ5nFKo9dNOc,4942
|
|
18
|
+
clue/common/dict_utils.py,sha256=BFemy28KakuduNFRPB-5c-OR2dQsm64dakXG2W4jdOE,3810
|
|
19
|
+
clue/common/exceptions.py,sha256=5ZGCneXbh3oPuysOajnM6HAvKsxvxdqtS2SpgvD3nJc,6363
|
|
20
|
+
clue/common/forge.py,sha256=9dlEqMLBJ4GhCUD2N9B3Z_3jhgfU_BYfSF8niA-k7Ds,5322
|
|
21
|
+
clue/common/json_utils.py,sha256=mOxPcBIctj3DlgdAnsUXrVyYYB4z4Do6BbqpCsTbJLo,340
|
|
22
|
+
clue/common/list_utils.py,sha256=DrpmdV5GJRw0XjoalPhP_368NUzh3L1pJT49ob6LovI,226
|
|
23
|
+
clue/common/logging/__init__.py,sha256=A5-l9RxguPOCkv4BFV-DqcqRtw-cDlIZfj6aH4g_VuA,9349
|
|
24
|
+
clue/common/logging/audit.py,sha256=5_sGZDwztXxK5ioUKU2ELLxlt_54TCXy9nlSstYF9ko,3668
|
|
25
|
+
clue/common/logging/format.py,sha256=X-YIPDJOiOSjF6-rA_arT-Q0d501zx2rfkwuUhrmbF4,1319
|
|
26
|
+
clue/common/regex.py,sha256=gUiuwor0UYf7mnzB-Tm2PLrOS80nN3bagRQmDlhcx9U,1865
|
|
27
|
+
clue/common/str_utils.py,sha256=IwMMMO9G2YtWjq9_-4DTzYrtkAEUZTPYGPiu6xKsOKE,7064
|
|
28
|
+
clue/common/swagger.py,sha256=cFx54xnfyUgQlB036RpmR2vk0gtVtKw7MX4p2hsp8i0,4569
|
|
29
|
+
clue/common/uid.py,sha256=WIPCItJsqAAN1DrsjhJxydC__Jovoj2dEhu3tGw_10Q,1386
|
|
30
|
+
clue/config.py,sha256=2g0DzixNMwRIzegQayZuR4VBBYnhsFCQC61BK1_hUXY,1290
|
|
31
|
+
clue/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
+
clue/constants/supported_types.py,sha256=F8EqngkscFryWLX10Ye_HmTRi55i2QbLeYg6SevD4h0,925
|
|
33
|
+
clue/cronjobs/__init__.py,sha256=EAD6qZ0nKyBLEMzhBSMGD87tdTo5nnCHbedLVSMZxDE,905
|
|
34
|
+
clue/cronjobs/plugins.py,sha256=-7cOU4sXx4FNn0rwDo7cpkJqykJiKvxXD6-nx0cG0Ps,1173
|
|
35
|
+
clue/error.py,sha256=pY1XmKbgsEdlZIBsx8DBL3AJRkVjyi5TKQbgNZa_v2A,4062
|
|
36
|
+
clue/gunicorn_config.py,sha256=YhSw2s952n9Pch78SznMiAF_Im6lkV-s-4TH_AB_d5E,981
|
|
37
|
+
clue/healthz.py,sha256=JLeaaCRZYLo526e7QCrWvhzPwphbdN1OmoFss5IdIt0,1297
|
|
38
|
+
clue/helper/discover.py,sha256=7CKEU6wzfz_sGaOqa8S5LbE0tuZ6EdG2HrckozwtQag,1983
|
|
39
|
+
clue/helper/headers.py,sha256=CWDAeJbaRIYZT0rd5DQqW3EXYSAneG2GaoT3z7tp92A,948
|
|
40
|
+
clue/helper/oauth.py,sha256=YgSrSvsckhOkX8kxq9P29HeLB_x52OTYxIYxoPIBjaI,4146
|
|
41
|
+
clue/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
|
+
clue/models/actions.py,sha256=v_J2Xas1TR8DRzbnBlx0PsTxWtQwyBtOMEhu3yELkaQ,9340
|
|
43
|
+
clue/models/config.py,sha256=MBtcZ0b5yBxFrCr_7EbxUgA7S8y2Wn3HWtOG8g751gU,18629
|
|
44
|
+
clue/models/fetchers.py,sha256=3ZwQovYG0t7fasV39noo8AVAC6TNqvW4-lvNwEkorCA,5265
|
|
45
|
+
clue/models/graph.py,sha256=BdpH0eHzdpRKuBudbS8SgKMXjZYb7q3j_g_Lr8OQV5U,6696
|
|
46
|
+
clue/models/model_list.py,sha256=CfZvDon4RX_fLVtEwyUJUcslvQgOwQiWKxL4IEYBqVU,1180
|
|
47
|
+
clue/models/network.py,sha256=dAuq5lSM1LUVE50BqaNeG2fbCSKX3MKyybTMNrCBul0,17704
|
|
48
|
+
clue/models/results/__init__.py,sha256=bVqhGf_XQwliXWWTFcgLNJxcWHcckWkseXWeG672JkM,1155
|
|
49
|
+
clue/models/results/base.py,sha256=X5J6jH3JadzeA5jwqPuXiv5QNT6AOcuAwMQRZx8xBeY,205
|
|
50
|
+
clue/models/results/graph.py,sha256=jztagJgfOZA0sLq3ClD5I1nanVLkAZOzfn358nbfZjg,817
|
|
51
|
+
clue/models/results/image.py,sha256=Dsk1_KIYUSE8NVf-J_tBc_ZedR7SQXILAosns0Ybig8,530
|
|
52
|
+
clue/models/results/status.py,sha256=g2mNPOybVu0PRVHUfSer8MNDEyW4BqgU3MePp6_vH5w,2066
|
|
53
|
+
clue/models/results/validation.py,sha256=vyNrTDGxw-RAGQHQaXE-IBUWz9ySoCbzLv-pGwqXlmM,2263
|
|
54
|
+
clue/models/selector.py,sha256=tmqdEf99odaWqJxBoc8Wh25pVfupF5ET7fqtGAUUsyc,2096
|
|
55
|
+
clue/models/utils.py,sha256=WRnTTy_d9jL00ix8k9iOXZfRBCbdaOrLWGYijrv6pMo,1711
|
|
56
|
+
clue/models/validators.py,sha256=UOY2LGWYsUhlLQZWShRVw2ppkcu92rw5DYCAiqiUXv0,530
|
|
57
|
+
clue/patched.py,sha256=yDtwaNo9dpuVajpDjHVfYsXNo0Wxb4SYNrS93WDX3Ko,140
|
|
58
|
+
clue/plugin/__init__.py,sha256=Xa2gk6sMEIc_-bogrLzL2iuYQOSUof91a9a5o6FmAQY,41188
|
|
59
|
+
clue/plugin/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
60
|
+
clue/plugin/helpers/central_server.py,sha256=_JOBJaLv-fsCfvu5FTajTyh7LevFCnOMQWrlvmyB7g4,1077
|
|
61
|
+
clue/plugin/helpers/email_render.py,sha256=yp3Isu6Uu1w424vkMrkQFcQLdCTi8bjN6P1egnvPv1I,8392
|
|
62
|
+
clue/plugin/helpers/token.py,sha256=D4g_f7Y-LX3pFFnAMMxfW79Hn5apoTdLRTedO8-5mXY,954
|
|
63
|
+
clue/plugin/helpers/trino.py,sha256=5MgkvYiGjUkQVMaAwaCmonZ5Ck9uu-WEwOsns190ulA,3776
|
|
64
|
+
clue/plugin/interactive.py,sha256=EgjoqPVPyUOfvGP7MzqojK3QLwLa9USHXuYdTrk7K2A,9393
|
|
65
|
+
clue/plugin/models.py,sha256=uxbaJh257bfdC9-nDpPIkyF5HN-8rkEGDprV_7AsZfw,625
|
|
66
|
+
clue/plugin/utils.py,sha256=ozym9Nof0sFjQeJAXEYan1A86m1yhYHP8hLYzLHuDYU,2721
|
|
67
|
+
clue/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
68
|
+
clue/remote/datatypes/__init__.py,sha256=gwaSYWRjAe--1TJm82sjH8bm6Jr0lWH2DJezu82vej8,3522
|
|
69
|
+
clue/remote/datatypes/cache.py,sha256=N8KIzhXh4n9Guod5hpUT-tK6KZlCGr3xYgUSW1-Dkww,1664
|
|
70
|
+
clue/remote/datatypes/events.py,sha256=DEiqxr3LD2ZO0yCij02Nsmz2na7iawPFWekOx_1FbKE,4041
|
|
71
|
+
clue/remote/datatypes/hash.py,sha256=e1RCHHwy5qLA8rKYsjQbADORC96472CPlUfgDYfmk7Y,6147
|
|
72
|
+
clue/remote/datatypes/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
73
|
+
clue/remote/datatypes/queues/comms.py,sha256=DEdvbAwPIZD2B4ey48EauXbvHZYVFqtBeSmnYMykPCA,1861
|
|
74
|
+
clue/remote/datatypes/set.py,sha256=s5Ic_8x2IjnaeV58aTvBf8w7bnswAVwAx5LBYBS926o,2842
|
|
75
|
+
clue/remote/datatypes/user_quota_tracker.py,sha256=Qtn5rUWk5qSc5abQ2yDFbKjHewldFAjNUh8JpQs4lXE,1941
|
|
76
|
+
clue/security/__init__.py,sha256=ouN0srOsGivxVFrEp69ziWGds8SmV0BmnNBQQ10s2A4,9143
|
|
77
|
+
clue/security/obo.py,sha256=tTaxe9y5OHSpWb2Szjy9zx3CwiYBlFmtPplMjtOO7Do,3369
|
|
78
|
+
clue/security/utils.py,sha256=4OQ1_kD7u5q4bjuHsxDN1VslgWez2bjKiT2RXIdOqmk,909
|
|
79
|
+
clue/services/action_service.py,sha256=38R1qVkN4B9HGkkA8c9Lixe5wfhJB6Z_eRYPbFoy-e4,7067
|
|
80
|
+
clue/services/auth_service.py,sha256=bbSciSIZYToQ5zOebtFIrcXPh7FF_hQFw0tMakqzXJ0,12577
|
|
81
|
+
clue/services/config_service.py,sha256=af3aL5Aollo9MEPEco-Hm0EBvfNVYKGlcwsR_UnSzxU,1237
|
|
82
|
+
clue/services/fetcher_service.py,sha256=Esu9JlhonnEi37TxuZ7vvsgZUp_yZvgZwpFYNXmHSIg,7705
|
|
83
|
+
clue/services/jwt_service.py,sha256=aW2U2nG-CtJcChnDuChqncXxeIt6N-aLvOQfzdRaYkc,7963
|
|
84
|
+
clue/services/lookup_service.py,sha256=f9BCDTtQ9sOqQy-k6xKJtRPDYFYe42j9zwdTljosTxU,29077
|
|
85
|
+
clue/services/type_service.py,sha256=-jYu080dUGk_dXMkMTFazywrhdIzRj2Vfl4-eAc8dOc,6517
|
|
86
|
+
clue/services/user_service.py,sha256=F7qFDeBWmEZIc0Zs56Wd4blGMJExT1W9ODb2W-XaJSg,5316
|
|
87
|
+
clue_api-1.0.0.dev7.dist-info/METADATA,sha256=t0OrcpKvFdemyb4tj0_iYoHPxLM_sjBAd591-R2gznY,4681
|
|
88
|
+
clue_api-1.0.0.dev7.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
89
|
+
clue_api-1.0.0.dev7.dist-info/entry_points.txt,sha256=__d_7oZxCcJh53agBVwmwU8oJ4YgFtUk7PdM1gFCzIU,263
|
|
90
|
+
clue_api-1.0.0.dev7.dist-info/licenses/LICENSE,sha256=teWkbPKSyJ7AiGvdZvy6Zp8GyrJDiTYW7BUaIBMKotE,1382
|
|
91
|
+
clue_api-1.0.0.dev7.dist-info/RECORD,,
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[console_scripts]
|
|
2
|
+
check_changes=build_scripts.check_changes:main
|
|
3
|
+
coverage_report=build_scripts.coverage_reports:main
|
|
4
|
+
last_success=build_scripts.last_success:main
|
|
5
|
+
server=clue.patched:main
|
|
6
|
+
test=build_scripts.run_tests:main
|
|
7
|
+
type_check=build_scripts.type_check:main
|
|
8
|
+
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 Crown Copyright, Government of Canada (Canadian Centre for Cyber Security / Communications Security Establishment)
|
|
4
|
+
|
|
5
|
+
Copyright title to all 3rd party software distributed with Clue is held by the respective copyright holders as noted in those files. Users are asked to read the 3rd Party Licenses referenced with those assets.
|
|
6
|
+
|
|
7
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
8
|
+
|
|
9
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
10
|
+
|
|
11
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|