stoobly-agent 0.34.9__py3-none-any.whl → 0.34.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stoobly_agent/__init__.py +1 -1
- stoobly_agent/app/api/proxy_controller.py +12 -10
- stoobly_agent/app/cli/config_cli.py +5 -3
- stoobly_agent/app/cli/helpers/handle_config_update_service.py +5 -0
- stoobly_agent/app/cli/snapshot_cli.py +85 -6
- stoobly_agent/app/models/adapters/python/response/mitmproxy_adapter.py +1 -1
- stoobly_agent/app/models/factories/resource/local_db/helpers/log.py +9 -7
- stoobly_agent/app/models/factories/resource/local_db/helpers/request_snapshot.py +5 -0
- stoobly_agent/app/models/factories/resource/local_db/helpers/scenario_snapshot.py +24 -0
- stoobly_agent/app/models/factories/resource/local_db/helpers/snapshot.py +18 -2
- stoobly_agent/app/models/factories/resource/local_db/helpers/tiebreak_scenario_request.py +6 -1
- stoobly_agent/app/models/factories/resource/local_db/query_param_adapter.py +9 -5
- stoobly_agent/app/models/factories/resource/local_db/request_adapter.py +1 -0
- stoobly_agent/app/proxy/handle_mock_service.py +16 -9
- stoobly_agent/app/proxy/handle_record_service.py +2 -2
- stoobly_agent/app/proxy/intercept_handler.py +7 -4
- stoobly_agent/app/proxy/mitmproxy/request_facade.py +4 -2
- stoobly_agent/app/proxy/mock/eval_fixtures_service.py +3 -1
- stoobly_agent/app/proxy/mock/hashed_request_decorator.py +10 -10
- stoobly_agent/app/proxy/mock/request_hasher.py +3 -1
- stoobly_agent/app/proxy/record/upload_request_service.py +5 -5
- stoobly_agent/app/proxy/replay/alias_resolver.py +5 -3
- stoobly_agent/app/proxy/replay/body_parser_service.py +1 -5
- stoobly_agent/app/proxy/replay/multipart.py +9 -27
- stoobly_agent/app/proxy/replay/trace_context.py +10 -9
- stoobly_agent/app/proxy/test/helpers/upload_test_service.py +5 -3
- stoobly_agent/app/proxy/utils/allowed_request_service.py +7 -5
- stoobly_agent/app/proxy/utils/request_handler.py +3 -1
- stoobly_agent/app/settings/__init__.py +32 -14
- stoobly_agent/cli.py +1 -1
- stoobly_agent/config/constants/custom_headers.py +1 -0
- stoobly_agent/config/data_dir.py +39 -21
- stoobly_agent/lib/api/body_param_names_resource.py +5 -3
- stoobly_agent/lib/api/endpoints_resource.py +5 -3
- stoobly_agent/lib/api/header_names_resource.py +5 -3
- stoobly_agent/lib/api/projects_resource.py +5 -3
- stoobly_agent/lib/api/query_param_names_resource.py +5 -3
- stoobly_agent/lib/api/requests_resource.py +5 -3
- stoobly_agent/lib/api/response_header_names_resource.py +5 -3
- stoobly_agent/lib/api/response_param_names_resource.py +5 -3
- stoobly_agent/lib/api/scenarios_resource.py +5 -3
- stoobly_agent/lib/api/stoobly_api.py +0 -1
- stoobly_agent/lib/api/test_responses_resource.py +3 -1
- stoobly_agent/lib/api/tests_resource.py +3 -1
- stoobly_agent/lib/api/users_resource.py +3 -1
- stoobly_agent/lib/cache.py +26 -9
- stoobly_agent/lib/logger.py +5 -2
- stoobly_agent/lib/utils/visitor.py +4 -3
- stoobly_agent/public/{13-es2015.220b4a1adf4cacb294e5.js → 13-es2015.343b0261a8b3b3f4a1fc.js} +1 -1
- stoobly_agent/public/{13-es5.220b4a1adf4cacb294e5.js → 13-es5.343b0261a8b3b3f4a1fc.js} +1 -1
- stoobly_agent/public/18-es2015.d3b430636a4d6f544d92.js +1 -0
- stoobly_agent/public/18-es5.d3b430636a4d6f544d92.js +1 -0
- stoobly_agent/public/35-es2015.f741ebce0bfc25f0ec99.js +1 -0
- stoobly_agent/public/35-es5.f741ebce0bfc25f0ec99.js +1 -0
- stoobly_agent/public/7-es2015.19ccb84e62e2ea874f53.js +1 -0
- stoobly_agent/public/7-es5.19ccb84e62e2ea874f53.js +1 -0
- stoobly_agent/public/9-es2015.b7bcad8238f58e214f03.js +1 -0
- stoobly_agent/public/9-es5.b7bcad8238f58e214f03.js +1 -0
- stoobly_agent/public/index.html +1 -1
- stoobly_agent/public/runtime-es2015.9addf49b79aca951b7e2.js +1 -0
- stoobly_agent/public/runtime-es5.9addf49b79aca951b7e2.js +1 -0
- stoobly_agent/test/app/cli/snapshot/snapshot_copy_test.py +56 -0
- stoobly_agent/test/app/cli/snapshot/snapshot_prune_test.py +2 -5
- stoobly_agent/test/app/cli/snapshot/snapshot_update_test.py +0 -1
- stoobly_agent/test/app/models/factories/resource/local_db/helpers/tiebreak_scenario_request_test.py +20 -2
- stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION +1 -1
- stoobly_agent/test/app/proxy/replay/body_parser_service_test.py +3 -3
- stoobly_agent/test/config/data_dir_test.py +4 -2
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/METADATA +2 -1
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/RECORD +73 -72
- stoobly_agent/public/18-es2015.10cdd5c608b10d90d19a.js +0 -1
- stoobly_agent/public/18-es5.10cdd5c608b10d90d19a.js +0 -1
- stoobly_agent/public/35-es2015.61a7ae8da93df94fab06.js +0 -1
- stoobly_agent/public/35-es5.61a7ae8da93df94fab06.js +0 -1
- stoobly_agent/public/7-es2015.c359dbb640e2af507221.js +0 -1
- stoobly_agent/public/7-es5.c359dbb640e2af507221.js +0 -1
- stoobly_agent/public/9-es2015.cfc1101139d6ae75731b.js +0 -1
- stoobly_agent/public/9-es5.cfc1101139d6ae75731b.js +0 -1
- stoobly_agent/public/runtime-es2015.08e65883d390cd16c15b.js +0 -1
- stoobly_agent/public/runtime-es5.08e65883d390cd16c15b.js +0 -1
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/LICENSE +0 -0
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/WHEEL +0 -0
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/entry_points.txt +0 -0
@@ -8,6 +8,8 @@ from typing import List, Dict, TypedDict, Union
|
|
8
8
|
from stoobly_agent.lib.logger import Logger, bcolors
|
9
9
|
from stoobly_agent.lib.utils.python_to_ruby_type import type_map
|
10
10
|
|
11
|
+
LOG_ID = 'RequestHasher'
|
12
|
+
|
11
13
|
class IgnoredParam(TypedDict):
|
12
14
|
inferred_type: str
|
13
15
|
query: str
|
@@ -107,7 +109,7 @@ class RequestHasher():
|
|
107
109
|
return self.type_map.get(str(val.__class__))
|
108
110
|
|
109
111
|
def __serialize_param(self, query, key: str, val):
|
110
|
-
Logger.instance().debug(f"{bcolors.OKBLUE}Serializing{bcolors.ENDC} {query}
|
112
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE}Serializing{bcolors.ENDC} {query} -> {val}")
|
111
113
|
|
112
114
|
if isinstance(val, bool):
|
113
115
|
# Ruby boolean are lower case
|
@@ -23,7 +23,7 @@ AGENT_STATUSES = {
|
|
23
23
|
'REQUESTS_MODIFIED': 'requests-modified'
|
24
24
|
}
|
25
25
|
|
26
|
-
LOG_ID = '
|
26
|
+
LOG_ID = 'Record'
|
27
27
|
NAMESPACE_FOLDER = 'stoobly'
|
28
28
|
|
29
29
|
def inject_upload_request(request_model: RequestModel, intercept_settings: InterceptSettings):
|
@@ -48,7 +48,7 @@ def inject_upload_request(request_model: RequestModel, intercept_settings: Inter
|
|
48
48
|
def upload_request(
|
49
49
|
request_model: RequestModel, intercept_settings: InterceptSettings, flow: MitmproxyHTTPFlow = None
|
50
50
|
):
|
51
|
-
Logger.instance().info(f"{bcolors.OKCYAN}
|
51
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Recording{bcolors.ENDC} {flow.request.url}")
|
52
52
|
|
53
53
|
joined_request = join_rewritten_request(flow, intercept_settings)
|
54
54
|
|
@@ -68,17 +68,17 @@ def upload_request(
|
|
68
68
|
|
69
69
|
if intercept_settings.settings.is_debug():
|
70
70
|
file_path = __debug_request(flow.request, joined_request.build())
|
71
|
-
Logger.instance().debug(f"
|
71
|
+
Logger.instance(LOG_ID).debug(f"Writing request to {file_path}")
|
72
72
|
elif not res:
|
73
73
|
file_path = __debug_request(flow.request, joined_request.build())
|
74
|
-
Logger.instance().error(f"Error: Failed to upload, writing request to {file_path}")
|
74
|
+
Logger.instance(LOG_ID).error(f"Error: Failed to upload, writing request to {file_path}")
|
75
75
|
|
76
76
|
return res
|
77
77
|
|
78
78
|
def upload_staged_request(
|
79
79
|
request: Request, request_model: RequestModel, project_key: str, scenario_key: str = None
|
80
80
|
):
|
81
|
-
Logger.instance().info(f"{bcolors.OKCYAN}
|
81
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Recording{bcolors.ENDC} {request.url}")
|
82
82
|
|
83
83
|
response = request.response
|
84
84
|
|
@@ -6,6 +6,8 @@ from stoobly_agent.lib.logger import Logger, bcolors
|
|
6
6
|
from stoobly_agent.lib.orm.trace import Trace
|
7
7
|
from stoobly_agent.lib.orm.trace_alias import TraceAlias
|
8
8
|
|
9
|
+
LOG_ID = 'Alias'
|
10
|
+
|
9
11
|
class AliasResolver():
|
10
12
|
|
11
13
|
def __init__(self, trace: Trace, strategy: alias_resolve_strategy.AliasResolveStrategy):
|
@@ -28,7 +30,7 @@ class AliasResolver():
|
|
28
30
|
trace_alias.assigned_to = value
|
29
31
|
trace_alias.save()
|
30
32
|
|
31
|
-
Logger.instance().info(f"{bcolors.OKBLUE}
|
33
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Assigned{bcolors.ENDC} {trace_alias.name}: {value} -> {trace_alias.value}")
|
32
34
|
|
33
35
|
def resolve_alias(self, alias_name: str, value: str):
|
34
36
|
trace_alias = self.__resolve_alias(alias_name, value)
|
@@ -51,7 +53,7 @@ class AliasResolver():
|
|
51
53
|
})
|
52
54
|
|
53
55
|
if trace_alias:
|
54
|
-
Logger.instance().info(f"{bcolors.OKGREEN}
|
56
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKGREEN}Created{bcolors.ENDC} alias {trace_alias.name}: {value}")
|
55
57
|
|
56
58
|
return trace_alias
|
57
59
|
|
@@ -114,4 +116,4 @@ class AliasResolver():
|
|
114
116
|
return str(value) if isinstance(value, list) or isinstance(value, dict) else value
|
115
117
|
|
116
118
|
def __log_resolved_aliases(self, trace_aliases):
|
117
|
-
trace_aliases.each(lambda trace_alias: Logger.instance().debug(f"\tResolved Trace Alias: {trace_alias.to_dict()}"))
|
119
|
+
trace_aliases.each(lambda trace_alias: Logger.instance(LOG_ID).debug(f"\tResolved Trace Alias: {trace_alias.to_dict()}"))
|
@@ -73,12 +73,8 @@ def parse_multipart_form_data(content, content_type) -> Dict[bytes, bytes]:
|
|
73
73
|
|
74
74
|
if not decoded_multipart:
|
75
75
|
return content
|
76
|
-
|
77
|
-
params_array = []
|
78
|
-
for ele in decoded_multipart:
|
79
|
-
params_array.append((decode(ele[0]), ele[1]))
|
80
76
|
|
81
|
-
return MultiDict(
|
77
|
+
return MultiDict(decoded_multipart)
|
82
78
|
|
83
79
|
def parse_www_form_urlencoded(content):
|
84
80
|
try:
|
@@ -1,7 +1,9 @@
|
|
1
|
+
import io
|
1
2
|
import mimetypes
|
2
3
|
import re
|
3
4
|
import pdb
|
4
5
|
|
6
|
+
from multipart import MultipartParser
|
5
7
|
from urllib.parse import quote
|
6
8
|
|
7
9
|
from mitmproxy.net.http import headers
|
@@ -64,31 +66,11 @@ def decode(hdrs, content):
|
|
64
66
|
except (KeyError, UnicodeError):
|
65
67
|
return
|
66
68
|
|
67
|
-
boundary_parts = content.split(b"--" + boundary)
|
68
|
-
if len(boundary_parts) == 0:
|
69
|
-
return
|
70
|
-
|
71
|
-
rx = re.compile(br'\bname="([^"]+)"')
|
72
69
|
r = []
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
# Continue parsing until we see just a line with CRLF
|
82
|
-
ar = parts[2:]
|
83
|
-
for i, ele in enumerate(ar):
|
84
|
-
if ele == CRLF:
|
85
|
-
value = b"".join(parts[3 + i:])
|
86
|
-
|
87
|
-
# Remove CRLF preceding the next boundary
|
88
|
-
length = len(value)
|
89
|
-
if value[length - 2:] == CRLF:
|
90
|
-
value = value[0:length - 2]
|
91
|
-
|
92
|
-
r.append((key, value))
|
93
|
-
break
|
94
|
-
return r
|
70
|
+
parser = MultipartParser(io.BytesIO(content), boundary=boundary)
|
71
|
+
for part in parser.parts():
|
72
|
+
if part.content_type.lower() == 'application/octet-stream':
|
73
|
+
r.append((part.name, part.raw))
|
74
|
+
else:
|
75
|
+
r.append((part.name, part.value))
|
76
|
+
return r
|
@@ -21,6 +21,7 @@ from stoobly_agent.lib.orm.trace_request import TraceRequest
|
|
21
21
|
from stoobly_agent.lib.utils import jmespath
|
22
22
|
|
23
23
|
AliasMap = Dict[str, RequestComponentName]
|
24
|
+
LOG_ID = 'Trace'
|
24
25
|
|
25
26
|
class TraceContext:
|
26
27
|
|
@@ -32,7 +33,7 @@ class TraceContext:
|
|
32
33
|
self.__requests: List[Request] = []
|
33
34
|
self.__remote_project_key: ProjectKey = None
|
34
35
|
|
35
|
-
Logger.instance().debug(f"Using trace {self.__trace.id}")
|
36
|
+
Logger.instance(LOG_ID).debug(f"Using trace {self.__trace.id}")
|
36
37
|
|
37
38
|
@property
|
38
39
|
def alias_resolve_strategy(self):
|
@@ -71,7 +72,7 @@ class TraceContext:
|
|
71
72
|
)
|
72
73
|
|
73
74
|
if endpoint:
|
74
|
-
Logger.instance().debug(f"\tMatched Endpoint: {endpoint}")
|
75
|
+
Logger.instance(LOG_ID).debug(f"\tMatched Endpoint: {endpoint}")
|
75
76
|
|
76
77
|
if self.alias_resolve_strategy != alias_resolve_strategy.NONE:
|
77
78
|
self.rewrite_request(request, endpoint)
|
@@ -133,7 +134,7 @@ class TraceContext:
|
|
133
134
|
request.path = '/' + '/'.join(path_segment_strings)
|
134
135
|
|
135
136
|
def __rewrite_handler(self, component_type: str, alias_name: str, name: str, value):
|
136
|
-
Logger.instance().info(f"{bcolors.OKCYAN}
|
137
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Rewriting{bcolors.ENDC} {component_type} alias {alias_name} {name} => {value}")
|
137
138
|
|
138
139
|
def __rewrite_headers(
|
139
140
|
self, request: Request, header_names: List[RequestComponentName], id_to_alias: AliasMap
|
@@ -217,7 +218,7 @@ class TraceContext:
|
|
217
218
|
'''
|
218
219
|
content = decode_response(response.content, response.headers.get('content-type'))
|
219
220
|
if not is_traversable(content):
|
220
|
-
return Logger.instance().debug('Skipping creating aliases, content is not traversable')
|
221
|
+
return Logger.instance(LOG_ID).debug('Skipping creating aliases, content is not traversable')
|
221
222
|
|
222
223
|
id_to_alias = {}
|
223
224
|
aliases = endpoint['aliases']
|
@@ -225,18 +226,18 @@ class TraceContext:
|
|
225
226
|
id_to_alias[_alias['id']] = _alias
|
226
227
|
|
227
228
|
response_param_names = endpoint['response_param_names']
|
228
|
-
Logger.instance().debug(f"\tBuilding Trace Aliases from: {response_param_names}")
|
229
|
+
Logger.instance(LOG_ID).debug(f"\tBuilding Trace Aliases from: {response_param_names}")
|
229
230
|
|
230
231
|
for response_param_name in response_param_names:
|
231
232
|
try:
|
232
233
|
values = self.__query_resolves_response(response_param_name, content)
|
233
|
-
Logger.instance().debug(f"\tValues: {values}")
|
234
|
+
Logger.instance(LOG_ID).debug(f"\tValues: {values}")
|
234
235
|
except Exception as e:
|
235
|
-
Logger.instance().error(e)
|
236
|
+
Logger.instance(LOG_ID).error(e)
|
236
237
|
continue
|
237
238
|
|
238
239
|
_alias: Alias = id_to_alias[response_param_name['alias_id']]
|
239
|
-
Logger.instance().debug(f"\tAlias: {_alias}")
|
240
|
+
Logger.instance(LOG_ID).debug(f"\tAlias: {_alias}")
|
240
241
|
|
241
242
|
for value in values:
|
242
243
|
if _alias and value:
|
@@ -292,4 +293,4 @@ class TraceContext:
|
|
292
293
|
'value': _alias.value,
|
293
294
|
})
|
294
295
|
|
295
|
-
tabulate_print(data, print_handler=Logger.instance().debug)
|
296
|
+
tabulate_print(data, print_handler=Logger.instance(LOG_ID).debug)
|
@@ -13,6 +13,8 @@ from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
|
|
13
13
|
from ...intercept_settings import InterceptSettings
|
14
14
|
from ...record.join_request_service import join_rewritten_request
|
15
15
|
|
16
|
+
LOG_ID = 'Test'
|
17
|
+
|
16
18
|
class UploadTestData(TypedDict):
|
17
19
|
expected_response: str
|
18
20
|
log: str
|
@@ -31,7 +33,7 @@ def inject_upload_test(
|
|
31
33
|
api = TestsResource(settings.remote.api_url, settings.remote.api_key)
|
32
34
|
|
33
35
|
if not intercept_settings:
|
34
|
-
intercept_settings = InterceptSettings(Settings.instance())
|
36
|
+
intercept_settings = InterceptSettings(Settings.instance(LOG_ID))
|
35
37
|
|
36
38
|
return lambda flow, **kwargs: upload_test(api, intercept_settings, flow, **kwargs)
|
37
39
|
|
@@ -43,14 +45,14 @@ def upload_test(
|
|
43
45
|
) -> Response:
|
44
46
|
joined_request = join_rewritten_request(flow, intercept_settings)
|
45
47
|
|
46
|
-
Logger.instance().info(f"{bcolors.OKCYAN}Uploading{bcolors.ENDC} test results for {joined_request.proxy_request.url()}")
|
48
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Uploading{bcolors.ENDC} test results for {joined_request.proxy_request.url()}")
|
47
49
|
|
48
50
|
raw_requests = joined_request.build()
|
49
51
|
|
50
52
|
# If report key is set, upload test to report
|
51
53
|
report_key = intercept_settings.report_key
|
52
54
|
if report_key:
|
53
|
-
Logger.instance().debug(f"Using report {report_key}")
|
55
|
+
Logger.instance(LOG_ID).debug(f"Using report {report_key}")
|
54
56
|
|
55
57
|
api.with_report_key(report_key, kwargs)
|
56
58
|
|
@@ -9,6 +9,8 @@ from stoobly_agent.app.settings.firewall_rule import FirewallRule
|
|
9
9
|
from stoobly_agent.config.constants import mock_policy, request_origin
|
10
10
|
from stoobly_agent.lib.logger import bcolors, Logger
|
11
11
|
|
12
|
+
LOG_ID = 'Firewall'
|
13
|
+
|
12
14
|
def get_active_mode_policy(request: MitmproxyRequest, intercept_settings: InterceptSettings):
|
13
15
|
if intercept_settings.request_origin == request_origin.CLI:
|
14
16
|
return intercept_settings.policy
|
@@ -39,7 +41,7 @@ def __request_excluded(request: MitmproxyRequest, exclude_rules: List[FirewallRu
|
|
39
41
|
rules = list(filter(lambda rule: method in rule.methods, exclude_rules))
|
40
42
|
patterns = list(map(lambda rule: rule.pattern, rules))
|
41
43
|
if __exclude(request, patterns):
|
42
|
-
Logger.instance().info(f"{bcolors.OKBLUE}{
|
44
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Excluding{bcolors.ENDC} {request.method} {request.url}")
|
43
45
|
return True
|
44
46
|
|
45
47
|
return False
|
@@ -54,12 +56,12 @@ def __request_included(request: MitmproxyRequest, include_rules: List[FirewallRu
|
|
54
56
|
# If there are include rules, but none that match the request's method,
|
55
57
|
# then we know that none of the include rules will match the request
|
56
58
|
if len(include_rules) > 0 and len(rules) == 0:
|
57
|
-
Logger.instance().info(f"{bcolors.OKBLUE}{
|
59
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Not Including{bcolors.ENDC} {request.method} {request.url}")
|
58
60
|
return False
|
59
61
|
|
60
62
|
patterns = list(map(lambda rule: rule.pattern, rules))
|
61
63
|
if not __include(request, patterns):
|
62
|
-
Logger.instance().info(f"{bcolors.OKBLUE}{
|
64
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Not Including{bcolors.ENDC} {request.method} {request.url}")
|
63
65
|
return False
|
64
66
|
|
65
67
|
return True
|
@@ -80,7 +82,7 @@ def __include(request: MitmproxyRequest, patterns: List[str]) -> bool:
|
|
80
82
|
if re.match(pattern, request.url):
|
81
83
|
return True
|
82
84
|
except re.error as e:
|
83
|
-
Logger.instance().error(f"RegExp error '{e}' for {pattern}")
|
85
|
+
Logger.instance(LOG_ID).error(f"RegExp error '{e}' for {pattern}")
|
84
86
|
return False
|
85
87
|
|
86
88
|
return False
|
@@ -94,7 +96,7 @@ def __exclude(request: MitmproxyRequest, patterns: List[str]) -> bool:
|
|
94
96
|
if re.match(pattern, request.url):
|
95
97
|
return True
|
96
98
|
except re.error as e:
|
97
|
-
Logger.instance().error(f"RegExp error '{e}' for {pattern}")
|
99
|
+
Logger.instance(LOG_ID).error(f"RegExp error '{e}' for {pattern}")
|
98
100
|
return True
|
99
101
|
|
100
102
|
return False
|
@@ -6,8 +6,10 @@ from urllib.parse import urlparse
|
|
6
6
|
from stoobly_agent.lib.logger import Logger
|
7
7
|
from stoobly_agent.lib.orm.utils.requests_response_builder import RequestsResponseBuilder
|
8
8
|
|
9
|
+
LOG_ID = 'ReversePorxy'
|
10
|
+
|
9
11
|
def reverse_proxy(request: MitmproxyRequest, service_url: str, options = {}):
|
10
|
-
Logger.instance().debug(
|
12
|
+
Logger.instance(LOG_ID).debug(service_url)
|
11
13
|
|
12
14
|
uri = urlparse(service_url)
|
13
15
|
|
@@ -18,10 +18,12 @@ from .proxy_settings import ProxySettings
|
|
18
18
|
from .remote_settings import RemoteSettings
|
19
19
|
from .ui_settings import UISettings
|
20
20
|
|
21
|
+
LOG_ID = 'Settings'
|
22
|
+
|
21
23
|
class Settings:
|
22
|
-
|
24
|
+
_instances = None
|
23
25
|
|
24
|
-
|
26
|
+
__data_dir: DataDir = None
|
25
27
|
|
26
28
|
__cli_settings = None
|
27
29
|
__proxy_settings = None
|
@@ -34,10 +36,11 @@ class Settings:
|
|
34
36
|
__load_lock = False
|
35
37
|
__watching = False
|
36
38
|
|
37
|
-
def __init__(self):
|
38
|
-
if Settings.
|
39
|
+
def __init__(self, data_dir_path: str = None):
|
40
|
+
if Settings._instances.get(data_dir_path):
|
39
41
|
raise RuntimeError('Call instance() instead')
|
40
42
|
|
43
|
+
self.__data_dir = DataDir.instance(data_dir_path)
|
41
44
|
self.__detect_paths()
|
42
45
|
|
43
46
|
# If the config does not exist, use template
|
@@ -47,11 +50,26 @@ class Settings:
|
|
47
50
|
self.__load_settings()
|
48
51
|
|
49
52
|
@classmethod
|
50
|
-
def instance(cls):
|
51
|
-
if cls.
|
52
|
-
cls.
|
53
|
+
def instance(cls, data_dir_path: str = None):
|
54
|
+
if not cls._instances:
|
55
|
+
cls._instances = {}
|
56
|
+
|
57
|
+
if not cls._instances.get(data_dir_path):
|
58
|
+
cls._instances[data_dir_path] = cls(data_dir_path)
|
53
59
|
|
54
|
-
return cls.
|
60
|
+
return cls._instances[data_dir_path]
|
61
|
+
|
62
|
+
@classmethod
|
63
|
+
def handle_chdir(cls):
|
64
|
+
'''
|
65
|
+
Reloads data dir to be relative to new working directory
|
66
|
+
'''
|
67
|
+
DataDir.handle_chdir()
|
68
|
+
|
69
|
+
if cls._instances and None in cls._instances:
|
70
|
+
del cls._instances[None]
|
71
|
+
|
72
|
+
return cls.instance()
|
55
73
|
|
56
74
|
### Statuses
|
57
75
|
|
@@ -62,7 +80,7 @@ class Settings:
|
|
62
80
|
|
63
81
|
@property
|
64
82
|
def cli(self):
|
65
|
-
return self.__cli_settings
|
83
|
+
return self.__cli_settings
|
66
84
|
|
67
85
|
@property
|
68
86
|
def ui(self):
|
@@ -107,8 +125,8 @@ class Settings:
|
|
107
125
|
if not self.__settings:
|
108
126
|
self.__load_settings()
|
109
127
|
|
110
|
-
return {
|
111
|
-
**self.__settings,
|
128
|
+
return {
|
129
|
+
**self.__settings,
|
112
130
|
**{ 'cli': self.__cli_settings.to_dict() },
|
113
131
|
**{ 'proxy': self.__proxy_settings.to_dict() },
|
114
132
|
**{ 'remote': self.__remote_settings.to_dict() },
|
@@ -165,7 +183,7 @@ class Settings:
|
|
165
183
|
copyfile(SourceDir.instance().settings_template_file_path, self.__settings_file_path)
|
166
184
|
|
167
185
|
def __detect_paths(self):
|
168
|
-
self.__settings_file_path = os.environ.get(env_vars.AGENT_CONFIG_PATH) or
|
186
|
+
self.__settings_file_path = os.environ.get(env_vars.AGENT_CONFIG_PATH) or self.__data_dir.settings_file_path
|
169
187
|
self.__schema_file_path = SourceDir.instance().schema_file_path
|
170
188
|
|
171
189
|
def __load_settings(self):
|
@@ -179,14 +197,14 @@ class Settings:
|
|
179
197
|
self.from_dict(settings)
|
180
198
|
except yaml.YAMLError as exc:
|
181
199
|
Logger.instance().error(exc)
|
182
|
-
|
200
|
+
|
183
201
|
def __reload_settings(self, event):
|
184
202
|
if not self.__load_lock:
|
185
203
|
from stoobly_agent.app.proxy.utils.publish_change_service import publish_change
|
186
204
|
|
187
205
|
self.__load_lock = True
|
188
206
|
|
189
|
-
Logger.instance().debug(
|
207
|
+
Logger.instance(LOG_ID).debug('Reloading settings')
|
190
208
|
self.__load_settings()
|
191
209
|
|
192
210
|
publish_change(statuses.SETTINGS_MODIFIED, self.__settings, sync=True)
|
stoobly_agent/cli.py
CHANGED
@@ -88,7 +88,7 @@ def init(**kwargs):
|
|
88
88
|
''')
|
89
89
|
@click.option('--confdir', default=os.path.join(os.path.expanduser('~'), '.mitmproxy'), help='Location of the default mitmproxy configuration files.')
|
90
90
|
@click.option('--connection-strategy', help=', '.join(CONNECTION_STRATEGIES), type=click.Choice(CONNECTION_STRATEGIES))
|
91
|
-
@click.option('--flow-detail', default='1', type=click.Choice(['1', '2', '3', '4']), help='''
|
91
|
+
@click.option('--flow-detail', default='1', type=click.Choice(['0', '1', '2', '3', '4']), help='''
|
92
92
|
The display detail level for flows in mitmdump: 0 (quiet) to 4 (very verbose).
|
93
93
|
0: no output
|
94
94
|
1: shortened request URL with response status code
|
@@ -5,6 +5,7 @@ REMOTE_PROJECT_KEY = 'X-Stoobly-Endpoints-Project-Id'
|
|
5
5
|
MOCK_POLICY = 'X-Mock-Policy'
|
6
6
|
MOCK_REQUEST_ID = 'X-Stoobly-Request-Id'
|
7
7
|
MOCK_REQUEST_ENDPOINT_ID = 'X-Stoobly-Request-Endpoint-Id'
|
8
|
+
MOCK_REQUEST_KEY = 'X-Stoobly-Request-Key'
|
8
9
|
DO_PROXY = 'X-Do-Proxy'
|
9
10
|
LIFECYCLE_HOOKS_PATH = 'X-Stoobly-Lifecycle-Hooks-Path'
|
10
11
|
PROJECT_KEY = 'X-Project-Key'
|
stoobly_agent/config/data_dir.py
CHANGED
@@ -8,34 +8,47 @@ class DataDir:
|
|
8
8
|
DB_FILE_NAME = 'stoobly_agent.sqlite3'
|
9
9
|
DB_VERSION_NAME = 'VERSION'
|
10
10
|
|
11
|
-
|
11
|
+
_instances = None
|
12
12
|
|
13
|
-
def __init__(self):
|
14
|
-
if DataDir.
|
13
|
+
def __init__(self, path: str = None):
|
14
|
+
if DataDir._instances.get(path):
|
15
15
|
raise RuntimeError('Call instance() instead')
|
16
16
|
else:
|
17
|
-
|
18
|
-
|
17
|
+
if path:
|
18
|
+
self.__data_dir_path = os.path.join(path, self.DATA_DIR_NAME)
|
19
|
+
else:
|
20
|
+
cwd = os.getcwd()
|
21
|
+
self.__data_dir_path = os.path.join(cwd, self.DATA_DIR_NAME)
|
22
|
+
|
23
|
+
# If the current working directory does not contain a .stoobly folder,
|
24
|
+
# then search in the parent directories until the home directory.
|
25
|
+
if not os.path.exists(self.__data_dir_path):
|
26
|
+
data_dir = self.find_data_dir(cwd)
|
27
|
+
|
28
|
+
if not data_dir:
|
29
|
+
self.__data_dir_path = os.path.join(os.path.expanduser('~'), self.DATA_DIR_NAME)
|
30
|
+
else:
|
31
|
+
self.__data_dir_path = data_dir
|
19
32
|
|
20
|
-
# If the current working directory does not contain a .stoobly folder,
|
21
|
-
# then search in the parent directories until the home directory.
|
22
33
|
if not os.path.exists(self.__data_dir_path):
|
23
|
-
|
34
|
+
os.makedirs(self.__data_dir_path, exist_ok=True)
|
24
35
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
36
|
+
@classmethod
|
37
|
+
def instance(cls, path: str = None):
|
38
|
+
if not cls._instances:
|
39
|
+
cls._instances = {}
|
29
40
|
|
30
|
-
|
31
|
-
|
41
|
+
if not cls._instances.get(path):
|
42
|
+
cls._instances[path] = cls(path)
|
43
|
+
|
44
|
+
return cls._instances[path]
|
32
45
|
|
33
46
|
@classmethod
|
34
|
-
def
|
35
|
-
if cls.
|
36
|
-
cls.
|
47
|
+
def handle_chdir(cls):
|
48
|
+
if cls._instances and None in cls._instances:
|
49
|
+
del cls._instances[None]
|
37
50
|
|
38
|
-
return cls.
|
51
|
+
return cls.instance()
|
39
52
|
|
40
53
|
@property
|
41
54
|
def path(self):
|
@@ -137,9 +150,14 @@ class DataDir:
|
|
137
150
|
def snapshosts_version_path(self):
|
138
151
|
return os.path.join(self.snapshots_dir_path, 'VERSION')
|
139
152
|
|
140
|
-
def remove(self):
|
141
|
-
if
|
142
|
-
|
153
|
+
def remove(self, directory_path = None):
|
154
|
+
if directory_path:
|
155
|
+
data_dir_path = os.path.join(directory_path, self.DATA_DIR_NAME)
|
156
|
+
else:
|
157
|
+
data_dir_path = self.path
|
158
|
+
|
159
|
+
if os.path.exists(data_dir_path):
|
160
|
+
shutil.rmtree(data_dir_path)
|
143
161
|
|
144
162
|
def create(self, directory_path = None):
|
145
163
|
if not directory_path:
|
@@ -7,6 +7,8 @@ from stoobly_agent.app.models.types import ParamNameCreateParams
|
|
7
7
|
from ..logger import Logger
|
8
8
|
from .endpoints_resource import EndpointsResource
|
9
9
|
|
10
|
+
LOG_ID = 'BodyParamNamesResource'
|
11
|
+
|
10
12
|
class BodyParamNamesResource(EndpointsResource):
|
11
13
|
BODY_PARAM_NAMES_ENDPOINT = 'body_param_names'
|
12
14
|
|
@@ -17,20 +19,20 @@ class BodyParamNamesResource(EndpointsResource):
|
|
17
19
|
def index(self, endpoint_id: int, query_params = {}) -> requests.Response:
|
18
20
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}/{self.BODY_PARAM_NAMES_ENDPOINT}"
|
19
21
|
|
20
|
-
Logger.instance().debug(f"{
|
22
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
21
23
|
|
22
24
|
return self.get(url, headers=self.default_headers, params=query_params)
|
23
25
|
|
24
26
|
def show(self, endpoint_id: int, body_param_name_id: int, query_params = {}) -> requests.Response:
|
25
27
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}/{self.BODY_PARAM_NAMES_ENDPOINT}/{body_param_name_id}"
|
26
28
|
|
27
|
-
Logger.instance().debug(f"{
|
29
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
28
30
|
|
29
31
|
return self.get(url, headers=self.default_headers, params=query_params)
|
30
32
|
|
31
33
|
def destroy(self, endpoint_id, body_param_name_id: int, query_params = {}) -> requests.Response:
|
32
34
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}/{self.BODY_PARAM_NAMES_ENDPOINT}/{body_param_name_id}"
|
33
35
|
|
34
|
-
Logger.instance().debug(f"{
|
36
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
35
37
|
|
36
38
|
return self.delete(url, headers=self.default_headers, params=query_params)
|
@@ -6,6 +6,8 @@ from ..logger import Logger
|
|
6
6
|
from .interfaces import EndpointsIndexQueryParams
|
7
7
|
from .stoobly_api import StooblyApi
|
8
8
|
|
9
|
+
LOG_ID = 'EndpointsResource'
|
10
|
+
|
9
11
|
class EndpointsResource(StooblyApi):
|
10
12
|
ENDPOINTS_ENDPOINT = 'endpoints'
|
11
13
|
|
@@ -16,20 +18,20 @@ class EndpointsResource(StooblyApi):
|
|
16
18
|
def index(self, **query_params: EndpointsIndexQueryParams) -> requests.Response:
|
17
19
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}"
|
18
20
|
|
19
|
-
Logger.instance().debug(f"{
|
21
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
20
22
|
|
21
23
|
return self.get(url, headers=self.default_headers, params=query_params)
|
22
24
|
|
23
25
|
def show(self, endpoint_id: int, **query_params) -> requests.Response:
|
24
26
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}"
|
25
27
|
|
26
|
-
Logger.instance().debug(f"{
|
28
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
27
29
|
|
28
30
|
return self.get(url, headers=self.default_headers, params=query_params)
|
29
31
|
|
30
32
|
def destroy(self, endpoint_id: int, **query_params) -> requests.Response:
|
31
33
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}"
|
32
34
|
|
33
|
-
Logger.instance().debug(f"{
|
35
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
34
36
|
|
35
37
|
return self.delete(url, headers=self.default_headers, params=query_params)
|
@@ -7,6 +7,8 @@ from stoobly_agent.app.models.types import HeaderNameCreateParams
|
|
7
7
|
from ..logger import Logger
|
8
8
|
from .endpoints_resource import EndpointsResource
|
9
9
|
|
10
|
+
LOG_ID = 'HeaderNamesResource'
|
11
|
+
|
10
12
|
class HeaderNamesResource(EndpointsResource):
|
11
13
|
HEADER_NAMES_ENDPOINT = 'header_names'
|
12
14
|
|
@@ -17,20 +19,20 @@ class HeaderNamesResource(EndpointsResource):
|
|
17
19
|
def index(self, endpoint_id: int, query_params = {}) -> requests.Response:
|
18
20
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}/{self.HEADER_NAMES_ENDPOINT}"
|
19
21
|
|
20
|
-
Logger.instance().debug(f"{
|
22
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
21
23
|
|
22
24
|
return self.get(url, headers=self.default_headers, params=query_params)
|
23
25
|
|
24
26
|
def show(self, endpoint_id: int, header_name_id: int, query_params = {}) -> requests.Response:
|
25
27
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}/{self.HEADER_NAMES_ENDPOINT}/{header_name_id}"
|
26
28
|
|
27
|
-
Logger.instance().debug(f"{
|
29
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
28
30
|
|
29
31
|
return self.get(url, headers=self.default_headers, params=query_params)
|
30
32
|
|
31
33
|
def destroy(self, endpoint_id: int, header_name_id: int, query_params = {}) -> requests.Response:
|
32
34
|
url = f"{self.service_url}/{self.ENDPOINTS_ENDPOINT}/{endpoint_id}/{self.HEADER_NAMES_ENDPOINT}/{header_name_id}"
|
33
35
|
|
34
|
-
Logger.instance().debug(f"{
|
36
|
+
Logger.instance(LOG_ID).debug(f"{url}?{urllib.parse.urlencode(query_params)}")
|
35
37
|
|
36
38
|
return self.delete(url, headers=self.default_headers, params=query_params)
|