stoobly-agent 1.4.2__py3-none-any.whl → 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stoobly_agent/__init__.py +1 -1
- stoobly_agent/app/cli/helpers/handle_mock_service.py +6 -2
- stoobly_agent/app/cli/helpers/request_facade.py +5 -1
- stoobly_agent/app/cli/scaffold/constants.py +1 -1
- stoobly_agent/app/cli/scaffold/docker/workflow/mock_decorator.py +1 -0
- stoobly_agent/app/cli/scaffold/service_workflow_validate_command.py +19 -19
- stoobly_agent/app/cli/scaffold/templates/app/.Dockerfile.context +1 -1
- stoobly_agent/app/cli/scaffold/templates/constants.py +3 -3
- stoobly_agent/app/cli/scaffold/templates/factory.py +5 -5
- stoobly_agent/app/cli/scaffold/templates/workflow/mock/bin/configure +1 -8
- stoobly_agent/app/cli/scaffold/templates/workflow/mock/fixtures.yml +1 -1
- stoobly_agent/app/cli/scaffold/templates/workflow/test/bin/configure +1 -8
- stoobly_agent/app/cli/scaffold/templates/workflow/test/fixtures.yml +1 -1
- stoobly_agent/app/cli/scaffold/workflow_command.py +3 -3
- stoobly_agent/app/cli/scaffold/workflow_create_command.py +2 -2
- stoobly_agent/app/cli/scaffold_cli.py +5 -5
- stoobly_agent/app/proxy/context.py +4 -0
- stoobly_agent/app/proxy/handle_mock_service.py +81 -54
- stoobly_agent/app/proxy/handle_record_service.py +15 -3
- stoobly_agent/app/proxy/handle_replay_service.py +44 -18
- stoobly_agent/app/proxy/handle_test_service.py +75 -16
- stoobly_agent/app/proxy/intercept_handler.py +11 -16
- stoobly_agent/app/proxy/intercept_settings.py +17 -4
- stoobly_agent/app/proxy/mitmproxy/request_facade.py +5 -2
- stoobly_agent/app/proxy/mitmproxy/response_facade.py +5 -4
- stoobly_agent/app/proxy/mock/eval_fixtures_service.py +78 -14
- stoobly_agent/app/proxy/mock/eval_request_service.py +2 -2
- stoobly_agent/app/proxy/record/join_request_service.py +7 -8
- stoobly_agent/app/proxy/record/upload_request_service.py +2 -2
- stoobly_agent/app/proxy/replay/replay_request_service.py +4 -4
- stoobly_agent/app/proxy/test/helpers/upload_test_service.py +2 -2
- stoobly_agent/app/proxy/utils/allowed_request_service.py +3 -3
- stoobly_agent/app/proxy/utils/response_handler.py +0 -2
- stoobly_agent/app/proxy/utils/rewrite.py +72 -0
- stoobly_agent/app/settings/constants/request_component.py +4 -1
- stoobly_agent/cli.py +35 -28
- stoobly_agent/config/constants/intercept_policy.py +2 -0
- stoobly_agent/config/constants/mock_policy.py +4 -2
- stoobly_agent/config/constants/record_policy.py +4 -2
- stoobly_agent/config/constants/replay_policy.py +4 -2
- stoobly_agent/public/{18-es2015.583f191cc7ad512ee262.js → 18-es2015.503207073756a9c8211a.js} +1 -1
- stoobly_agent/public/{18-es5.583f191cc7ad512ee262.js → 18-es5.503207073756a9c8211a.js} +1 -1
- stoobly_agent/public/index.html +1 -1
- stoobly_agent/public/{main-es2015.2cc16523aa3fcaba51e5.js → main-es2015.d682619f3d6d53d64c6a.js} +1 -1
- stoobly_agent/public/{main-es5.2cc16523aa3fcaba51e5.js → main-es5.d682619f3d6d53d64c6a.js} +1 -1
- stoobly_agent/public/{runtime-es2015.b914470164e4d6e75d96.js → runtime-es2015.8c1efed946fc02c923fc.js} +1 -1
- stoobly_agent/public/{runtime-es5.b914470164e4d6e75d96.js → runtime-es5.8c1efed946fc02c923fc.js} +1 -1
- stoobly_agent/test/app/cli/helpers/openapi_endpoint_adapter_test.py +2 -1
- stoobly_agent/test/app/cli/scaffold/e2e_test.py +2 -2
- stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION +1 -1
- stoobly_agent/test/app/proxy/mock/eval_fixtures_service_test.py +140 -71
- stoobly_agent/test/cli/lifecycle_hooks_test.py +66 -0
- stoobly_agent/test/cli/mock_test.py +53 -29
- stoobly_agent/test/cli/record_test.py +67 -0
- stoobly_agent/test/mock_data/lifecycle_hooks.py +35 -0
- {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/LICENSE +1 -1
- {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/METADATA +7 -12
- {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/RECORD +62 -58
- /stoobly_agent/app/cli/scaffold/templates/workflow/mock/{fixtures/.keep → public/.gitignore} +0 -0
- /stoobly_agent/app/cli/scaffold/templates/workflow/test/{fixtures/.keep → public/.gitignore} +0 -0
- {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/WHEEL +0 -0
- {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/entry_points.txt +0 -0
@@ -1,7 +1,9 @@
|
|
1
1
|
import pdb
|
2
2
|
|
3
|
+
from copy import deepcopy
|
3
4
|
from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
|
4
5
|
|
6
|
+
from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
|
5
7
|
from stoobly_agent.app.proxy.replay.body_parser_service import encode_response
|
6
8
|
from stoobly_agent.app.proxy.replay.context import ReplayContext
|
7
9
|
from stoobly_agent.app.proxy.utils.request_handler import build_response
|
@@ -12,35 +14,53 @@ from stoobly_agent.lib.api.endpoints_resource import EndpointsResource
|
|
12
14
|
from stoobly_agent.lib.api.interfaces.tests import TestShowResponse
|
13
15
|
from stoobly_agent.lib.logger import Logger
|
14
16
|
|
15
|
-
from .handle_mock_service import
|
16
|
-
from .handle_replay_service import
|
17
|
+
from .handle_mock_service import handle_request_mock_generic_without_rewrite
|
18
|
+
from .handle_replay_service import handle_request_replay_without_rewrite
|
17
19
|
from .mock.context import MockContext
|
18
20
|
from .test.helpers.test_results_builder import TestResultsBuilder
|
19
21
|
from .test.helpers.upload_test_service import inject_upload_test
|
20
22
|
from .test.context_abc import TestContextABC as TestContext
|
21
23
|
from .test.test_service import test
|
24
|
+
from .utils.rewrite import rewrite_request, rewrite_response, rewrite_request_response
|
22
25
|
|
23
26
|
LOG_ID = 'HandleTest'
|
24
27
|
|
28
|
+
###
|
29
|
+
#
|
30
|
+
# 1. Rewrites test request
|
31
|
+
# 2. BEFORE_REPLAY gets triggered
|
32
|
+
#
|
25
33
|
def handle_request_test(context: ReplayContext) -> None:
|
26
|
-
|
34
|
+
__rewrite_request(context)
|
35
|
+
handle_request_replay_without_rewrite(context)
|
27
36
|
|
28
37
|
###
|
29
38
|
#
|
30
|
-
#
|
39
|
+
# 1. Rewrites test response (response from live service)
|
40
|
+
# 2. AFTER_REPLAY gets triggered
|
41
|
+
# 3. Uses rewritten test request to obtain mock response
|
42
|
+
# 4. BEFORE_MOCK gets triggered
|
43
|
+
# 5. AFTER_MOCK gets triggered
|
44
|
+
# 6. BEFORE_TEST gets triggered
|
45
|
+
# 7. Tests against rewritten test response and mock response (expected response)
|
46
|
+
# 8. Rewrites a copy of request and response
|
47
|
+
# 9. BEFORE_RECORD gets triggered (if not from CLI)
|
48
|
+
# 10. AFTER_RECORD gets triggered (if not from CLI)
|
49
|
+
# 11. AFTER_TEST gets triggered
|
31
50
|
#
|
32
51
|
def handle_response_test(context: ReplayContext) -> None:
|
33
52
|
from .test.context import TestContext
|
34
53
|
|
35
54
|
flow: MitmproxyHTTPFlow = context.flow
|
36
|
-
intercept_settings = context.intercept_settings
|
37
|
-
|
38
55
|
disable_transfer_encoding(flow.response)
|
39
56
|
|
40
|
-
|
41
|
-
|
57
|
+
__rewrite_response(context)
|
58
|
+
__test_hook(lifecycle_hooks.AFTER_REPLAY, context)
|
42
59
|
|
43
|
-
|
60
|
+
intercept_settings = context.intercept_settings
|
61
|
+
|
62
|
+
# At this point, the request may already been rewritten during replay, do not rewrite again
|
63
|
+
handle_request_mock_generic_without_rewrite(
|
44
64
|
MockContext(flow, intercept_settings),
|
45
65
|
failure=lambda mock_context: __handle_mock_failure(TestContext(context, mock_context)),
|
46
66
|
#infer=intercept_settings.test_strategy == test_strategy.FUZZY, # For fuzzy testing we can use an inferred response
|
@@ -53,8 +73,10 @@ def __decorate_test_id(flow: MitmproxyHTTPFlow, test_response: TestShowResponse)
|
|
53
73
|
|
54
74
|
def __handle_mock_success(test_context: TestContext) -> None:
|
55
75
|
flow: MitmproxyHTTPFlow = test_context.flow
|
56
|
-
settings = Settings.instance()
|
57
76
|
|
77
|
+
__test_hook(lifecycle_hooks.AFTER_MOCK, test_context.mock_context)
|
78
|
+
|
79
|
+
settings: Settings = Settings.instance()
|
58
80
|
test_context.with_endpoints_resource(EndpointsResource(settings.remote.api_url, settings.remote.api_key))
|
59
81
|
|
60
82
|
__test_hook(lifecycle_hooks.BEFORE_TEST, test_context)
|
@@ -84,13 +106,8 @@ def __handle_mock_success(test_context: TestContext) -> None:
|
|
84
106
|
if not is_cli or test_context.save:
|
85
107
|
# Re-serialize expected response since it was rewritten
|
86
108
|
upload_test_data['expected_response'] = encode_response(expected, test_context.expected_response.content_type)
|
87
|
-
upload_test = inject_upload_test(None, intercept_settings)
|
88
109
|
|
89
|
-
|
90
|
-
res = upload_test(
|
91
|
-
flow,
|
92
|
-
**upload_test_data
|
93
|
-
)
|
110
|
+
res = __record_handler(test_context, upload_test_data)
|
94
111
|
|
95
112
|
if is_cli:
|
96
113
|
# If the origin was from a CLI, send test ID in response header
|
@@ -136,6 +153,48 @@ def __override_response(flow: MitmproxyHTTPFlow, content: bytes):
|
|
136
153
|
flow.response.set_content(content)
|
137
154
|
flow.response.status_code = 200
|
138
155
|
|
156
|
+
def __record_handler(context: TestContext, upload_test_data):
|
157
|
+
flow = context.flow
|
158
|
+
flow_copy = deepcopy(flow) # Deep copy flow to prevent response modifications from persisting
|
159
|
+
intercept_settings = context.intercept_settings
|
160
|
+
|
161
|
+
context.flow = flow_copy
|
162
|
+
|
163
|
+
# Since we are "uploading" the request, use record_write_rules
|
164
|
+
rewrite_request_response(flow_copy, intercept_settings.record_rewrite_rules)
|
165
|
+
__test_hook(lifecycle_hooks.BEFORE_RECORD, context)
|
166
|
+
|
167
|
+
# Commit test to API
|
168
|
+
upload_test = inject_upload_test(None, intercept_settings)
|
169
|
+
res = upload_test(
|
170
|
+
flow_copy, **upload_test_data
|
171
|
+
)
|
172
|
+
|
173
|
+
__test_hook(lifecycle_hooks.AFTER_RECORD, context)
|
174
|
+
context.flow = flow
|
175
|
+
|
176
|
+
return res
|
177
|
+
|
178
|
+
def __rewrite_request(replay_context: ReplayContext):
|
179
|
+
"""
|
180
|
+
Before replaying a request, see if the request needs to be rewritten
|
181
|
+
"""
|
182
|
+
intercept_settings: InterceptSettings = replay_context.intercept_settings
|
183
|
+
rewrite_rules = intercept_settings.test_rewrite_rules
|
184
|
+
|
185
|
+
if len(rewrite_rules) > 0:
|
186
|
+
rewrite_request(replay_context.flow, rewrite_rules)
|
187
|
+
|
188
|
+
def __rewrite_response(replay_context: ReplayContext):
|
189
|
+
"""
|
190
|
+
After replaying a request, see if the request needs to be rewritten
|
191
|
+
"""
|
192
|
+
intercept_settings: InterceptSettings = replay_context.intercept_settings
|
193
|
+
rewrite_rules = intercept_settings.test_rewrite_rules
|
194
|
+
|
195
|
+
if len(rewrite_rules) > 0:
|
196
|
+
rewrite_response(replay_context.flow, rewrite_rules)
|
197
|
+
|
139
198
|
def __test_hook(hook: str, context: TestContext):
|
140
199
|
intercept_settings = context.intercept_settings
|
141
200
|
lifecycle_hooks_module = intercept_settings.lifecycle_hooks
|
@@ -33,6 +33,7 @@ def request(flow: MitmproxyHTTPFlow):
|
|
33
33
|
if not intercept_settings.active:
|
34
34
|
return
|
35
35
|
|
36
|
+
__disable_web_cache(request)
|
36
37
|
__intercept_hook(lifecycle_hooks.BEFORE_REQUEST, flow, intercept_settings)
|
37
38
|
|
38
39
|
active_mode = intercept_settings.mode
|
@@ -42,7 +43,7 @@ def request(flow: MitmproxyHTTPFlow):
|
|
42
43
|
context = MockContext(flow, intercept_settings)
|
43
44
|
handle_request_mock(context)
|
44
45
|
elif active_mode == mode.RECORD:
|
45
|
-
|
46
|
+
pass
|
46
47
|
elif active_mode == mode.REPLAY:
|
47
48
|
context = ReplayContext(flow, intercept_settings)
|
48
49
|
handle_request_replay(context)
|
@@ -65,35 +66,29 @@ def response(flow: MitmproxyHTTPFlow):
|
|
65
66
|
if not intercept_settings.active:
|
66
67
|
return
|
67
68
|
|
68
|
-
__intercept_hook(lifecycle_hooks.BEFORE_RESPONSE, flow, intercept_settings)
|
69
|
-
|
70
69
|
active_mode = intercept_settings.mode
|
71
70
|
|
72
71
|
if active_mode == mode.MOCK:
|
73
72
|
context = MockContext(flow, intercept_settings)
|
74
|
-
|
73
|
+
handle_response_mock(context)
|
75
74
|
elif active_mode == mode.RECORD:
|
76
75
|
context = RecordContext(flow, intercept_settings)
|
77
|
-
|
76
|
+
handle_response_record(context)
|
78
77
|
elif active_mode == mode.REPLAY:
|
79
78
|
context = ReplayContext(flow, intercept_settings)
|
80
|
-
|
79
|
+
handle_response_replay(context)
|
81
80
|
elif active_mode == mode.TEST:
|
82
81
|
context = ReplayContext(flow, intercept_settings)
|
83
|
-
|
82
|
+
handle_response_test(context)
|
83
|
+
|
84
|
+
__intercept_hook(lifecycle_hooks.BEFORE_RESPONSE, flow, intercept_settings)
|
84
85
|
|
85
86
|
### PRIVATE
|
86
87
|
|
88
|
+
# Prevent 304 status
|
89
|
+
# Because this header will get recorded, should add during mocking as well in the case where headers are used for matching
|
87
90
|
def __disable_web_cache(request: MitmproxyRequest) -> None:
|
88
|
-
request.headers['Cache-Control'] = 'no-cache, no-store
|
89
|
-
request.headers['Expires'] = '0'
|
90
|
-
request.headers['Pragma'] = 'no-cache'
|
91
|
-
|
92
|
-
if 'IF-NONE-MATCH' in request.headers:
|
93
|
-
del request.headers['IF-NONE-MATCH']
|
94
|
-
|
95
|
-
if 'IF-MODIFIED-SINCE' in request.headers:
|
96
|
-
del request.headers['IF-MODIFIED-SINCE']
|
91
|
+
request.headers['Cache-Control'] = 'no-cache, no-store'
|
97
92
|
|
98
93
|
# Fix issue where multi-value cookies become comma separated
|
99
94
|
def __patch_cookie(request: MitmproxyRequest):
|
@@ -41,6 +41,11 @@ class InterceptSettings:
|
|
41
41
|
self.__response_fixtures = None
|
42
42
|
self.__initialize_response_fixtures()
|
43
43
|
|
44
|
+
self._mock_rewrite_rules = None
|
45
|
+
self._record_rewrite_rules = None
|
46
|
+
self._replay_rewrite_rules = None
|
47
|
+
self._test_rewrite_rules = None
|
48
|
+
|
44
49
|
@property
|
45
50
|
def settings(self):
|
46
51
|
return self.__settings
|
@@ -200,19 +205,27 @@ class InterceptSettings:
|
|
200
205
|
|
201
206
|
@property
|
202
207
|
def record_rewrite_rules(self) -> List[RewriteRule]:
|
203
|
-
|
208
|
+
if not self._record_rewrite_rules:
|
209
|
+
self._record_rewrite_rules = self.__select_rewrite_rules(mode.RECORD)
|
210
|
+
return self._record_rewrite_rules
|
204
211
|
|
205
212
|
@property
|
206
213
|
def mock_rewrite_rules(self) -> List[RewriteRule]:
|
207
|
-
|
214
|
+
if not self._mock_rewrite_rules:
|
215
|
+
self._mock_rewrite_rules = self.__select_rewrite_rules(mode.MOCK)
|
216
|
+
return self._mock_rewrite_rules
|
208
217
|
|
209
218
|
@property
|
210
219
|
def replay_rewrite_rules(self) -> List[RewriteRule]:
|
211
|
-
|
220
|
+
if not self._replay_rewrite_rules:
|
221
|
+
self._replay_rewrite_rules = self.__select_rewrite_rules(mode.REPLAY)
|
222
|
+
return self._replay_rewrite_rules
|
212
223
|
|
213
224
|
@property
|
214
225
|
def test_rewrite_rules(self) -> List[RewriteRule]:
|
215
|
-
|
226
|
+
if not self._test_rewrite_rules:
|
227
|
+
self._test_rewrite_rules = self.__select_rewrite_rules(mode.TEST)
|
228
|
+
return self._test_rewrite_rules
|
216
229
|
|
217
230
|
@property
|
218
231
|
def upstream_url(self):
|
@@ -239,7 +239,7 @@ class MitmproxyRequestFacade(Request):
|
|
239
239
|
content_type = self.content_type
|
240
240
|
parsed_content = self.__body.get(content_type)
|
241
241
|
|
242
|
-
if not
|
242
|
+
if not self.__is_iterable(parsed_content):
|
243
243
|
content_type = 'application/json'
|
244
244
|
self.request.headers['content-type'] = content_type
|
245
245
|
parsed_content = {}
|
@@ -265,4 +265,7 @@ class MitmproxyRequestFacade(Request):
|
|
265
265
|
|
266
266
|
_request_headers.pop(name)
|
267
267
|
|
268
|
-
return _request_headers
|
268
|
+
return _request_headers
|
269
|
+
|
270
|
+
def __is_iterable(self, v):
|
271
|
+
return isinstance(v, dict) or isinstance(v, multidict.MultiDictView) or isinstance(v, list)
|
@@ -95,10 +95,8 @@ class MitmproxyResponseFacade(Response):
|
|
95
95
|
content_type = self.content_type
|
96
96
|
parsed_content = self.__body.get(content_type)
|
97
97
|
|
98
|
-
if not
|
99
|
-
|
100
|
-
self.response.headers['content-type'] = content_type
|
101
|
-
parsed_content = {}
|
98
|
+
if not self.__is_iterable(parsed_content):
|
99
|
+
return
|
102
100
|
|
103
101
|
self.__apply_rewrites(parsed_content, rewrites, handler)
|
104
102
|
self.__body.set(parsed_content, content_type)
|
@@ -122,3 +120,6 @@ class MitmproxyResponseFacade(Response):
|
|
122
120
|
_response_headers.pop(name)
|
123
121
|
|
124
122
|
return _response_headers
|
123
|
+
|
124
|
+
def __is_iterable(self, v):
|
125
|
+
return isinstance(v, dict) or isinstance(v, multidict.MultiDictView) or isinstance(v, list)
|
@@ -1,3 +1,4 @@
|
|
1
|
+
import mimetypes
|
1
2
|
import os
|
2
3
|
import pdb
|
3
4
|
import re
|
@@ -5,6 +6,7 @@ import re
|
|
5
6
|
from io import BytesIO
|
6
7
|
from mitmproxy.http import Request as MitmproxyRequest
|
7
8
|
from requests import Response
|
9
|
+
from requests.structures import CaseInsensitiveDict
|
8
10
|
from typing import Union
|
9
11
|
|
10
12
|
from stoobly_agent.lib.logger import bcolors, Logger
|
@@ -19,45 +21,88 @@ class Options():
|
|
19
21
|
|
20
22
|
def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Response, None]:
|
21
23
|
fixture_path = None
|
22
|
-
headers =
|
24
|
+
headers = CaseInsensitiveDict()
|
25
|
+
status_code = 200
|
23
26
|
|
24
27
|
response_fixtures = options.get('response_fixtures')
|
25
|
-
fixture = __eval_response_fixtures(request, response_fixtures)
|
28
|
+
fixture: dict = __eval_response_fixtures(request, response_fixtures)
|
26
29
|
|
27
30
|
if not fixture:
|
28
31
|
public_directory_path = options.get('public_directory_path')
|
29
32
|
|
30
|
-
if
|
31
|
-
|
33
|
+
if not public_directory_path:
|
34
|
+
return
|
32
35
|
|
33
|
-
|
34
|
-
|
36
|
+
request_path = 'index' if request.path == '/' else request.path
|
37
|
+
_fixture_path = os.path.join(public_directory_path, request_path.lstrip('/'))
|
38
|
+
if request.headers.get('accept'):
|
39
|
+
fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
|
40
|
+
|
41
|
+
if not fixture_path:
|
42
|
+
fixture_path = _fixture_path
|
43
|
+
if not os.path.isfile(fixture_path):
|
44
|
+
return
|
35
45
|
else:
|
36
46
|
fixture_path = fixture.get('path')
|
37
|
-
|
47
|
+
if not fixture_path or not os.path.isfile(fixture_path):
|
48
|
+
return
|
38
49
|
|
39
|
-
|
40
|
-
|
50
|
+
_headers = fixture.get('headers')
|
51
|
+
headers = CaseInsensitiveDict(_headers if isinstance(_headers, dict) else {})
|
52
|
+
|
53
|
+
if fixture.get('status_code'):
|
54
|
+
status_code = fixture.get('status_code')
|
41
55
|
|
42
56
|
with open(fixture_path, 'rb') as fp:
|
43
57
|
response = Response()
|
44
58
|
|
45
|
-
response.status_code =
|
59
|
+
response.status_code = status_code
|
46
60
|
response.raw = BytesIO(fp.read())
|
47
61
|
response.headers = headers
|
48
62
|
|
63
|
+
if not response.headers.get('content-type'):
|
64
|
+
content_type = __guess_content_type(fixture_path)
|
65
|
+
if content_type:
|
66
|
+
response.headers['content-type'] = content_type
|
67
|
+
|
49
68
|
Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE}Resolved{bcolors.ENDC} fixture {fixture_path}")
|
50
69
|
|
51
70
|
return response
|
52
71
|
|
72
|
+
def __guess_content_type(file_path):
|
73
|
+
file_extension = os.path.splitext(file_path)[1]
|
74
|
+
if not file_extension:
|
75
|
+
return
|
76
|
+
return mimetypes.types_map.get(file_extension)
|
77
|
+
|
78
|
+
def __guess_file_path(file_path, content_type):
|
79
|
+
file_extension = os.path.splitext(file_path)[1]
|
80
|
+
if file_extension:
|
81
|
+
return file_path
|
82
|
+
|
83
|
+
if not content_type:
|
84
|
+
return
|
85
|
+
|
86
|
+
content_types = __parse_accept_header(content_type)
|
87
|
+
|
88
|
+
for content_type in content_types:
|
89
|
+
file_extension = mimetypes.guess_extension(content_type)
|
90
|
+
|
91
|
+
if not file_extension:
|
92
|
+
continue
|
93
|
+
|
94
|
+
_file_path = f"{file_path}{file_extension}"
|
95
|
+
if os.path.isfile(_file_path):
|
96
|
+
return _file_path
|
97
|
+
|
53
98
|
def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtures):
|
54
|
-
if not response_fixtures:
|
99
|
+
if not isinstance(response_fixtures, dict):
|
55
100
|
return
|
56
101
|
|
57
102
|
method = request.method
|
58
103
|
routes = response_fixtures.get(method)
|
59
104
|
|
60
|
-
if not routes:
|
105
|
+
if not isinstance(routes, dict):
|
61
106
|
return
|
62
107
|
|
63
108
|
for path_pattern in routes:
|
@@ -65,7 +110,26 @@ def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtu
|
|
65
110
|
continue
|
66
111
|
|
67
112
|
fixture = routes[path_pattern]
|
113
|
+
if not isinstance(fixture, dict):
|
114
|
+
continue
|
115
|
+
|
68
116
|
path = fixture.get('path')
|
69
117
|
|
70
|
-
if path
|
71
|
-
return fixture
|
118
|
+
if path:
|
119
|
+
return fixture
|
120
|
+
|
121
|
+
def __parse_accept_header(accept_header):
|
122
|
+
types = []
|
123
|
+
for item in accept_header.split(","):
|
124
|
+
parts = item.split(";")
|
125
|
+
content_type = parts[0].strip()
|
126
|
+
q_value = 1.0 # Default quality value
|
127
|
+
if len(parts) > 1 and parts[1].strip().startswith("q="):
|
128
|
+
try:
|
129
|
+
q_value = float(parts[1].strip()[2:])
|
130
|
+
except ValueError:
|
131
|
+
pass # Keep default q_value if parsing fails
|
132
|
+
types.append((content_type, q_value))
|
133
|
+
|
134
|
+
# Sort by quality factor in descending order
|
135
|
+
return [content_type for content_type, _ in sorted(types, key=lambda x: x[1], reverse=True)]
|
@@ -149,9 +149,9 @@ def __filter_by_match_rules(request: MitmproxyRequest, match_rules: List[MatchRu
|
|
149
149
|
method = request.method.upper()
|
150
150
|
|
151
151
|
keep = {
|
152
|
-
request_component.BODY_PARAM:
|
152
|
+
request_component.BODY_PARAM: False,
|
153
153
|
request_component.HEADER: False,
|
154
|
-
request_component.QUERY_PARAM:
|
154
|
+
request_component.QUERY_PARAM: False,
|
155
155
|
}
|
156
156
|
|
157
157
|
match_rules = list(filter(lambda rule: method in rule.methods, match_rules))
|
@@ -1,10 +1,14 @@
|
|
1
1
|
import pdb
|
2
|
+
|
2
3
|
from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
|
4
|
+
from typing import List
|
3
5
|
|
4
6
|
from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
|
7
|
+
from stoobly_agent.app.settings.rewrite_rule import RewriteRule
|
5
8
|
|
6
9
|
from ..mitmproxy.request_facade import MitmproxyRequestFacade
|
7
10
|
from ..mitmproxy.response_facade import MitmproxyResponseFacade
|
11
|
+
from ..utils.rewrite import rewrite_request_response
|
8
12
|
from .joined_request import JoinedRequest
|
9
13
|
from .proxy_request import ProxyRequest
|
10
14
|
|
@@ -18,15 +22,10 @@ def join_request(
|
|
18
22
|
# Create JoinedRequest
|
19
23
|
return JoinedRequest(proxy_request).with_response(adapted_response)
|
20
24
|
|
21
|
-
def
|
22
|
-
|
25
|
+
def join_request_from_flow(
|
26
|
+
flow: MitmproxyHTTPFlow, intercept_settings: InterceptSettings
|
27
|
+
) -> JoinedRequest:
|
23
28
|
request = MitmproxyRequestFacade(flow.request)
|
24
|
-
|
25
|
-
# Adapt flow.response
|
26
29
|
response = MitmproxyResponseFacade(flow.response)
|
27
|
-
rewrite_rules = intercept_settings.record_rewrite_rules
|
28
|
-
|
29
|
-
request.with_parameter_rules(rewrite_rules).with_url_rules(rewrite_rules).rewrite()
|
30
|
-
response.with_parameter_rules(rewrite_rules, request).rewrite()
|
31
30
|
|
32
31
|
return join_request(request, response, intercept_settings)
|
@@ -17,7 +17,7 @@ from stoobly_agent.lib.logger import Logger, bcolors
|
|
17
17
|
from stoobly_agent.lib.orm.request import Request
|
18
18
|
|
19
19
|
from ..utils.publish_change_service import publish_requests_modified
|
20
|
-
from .join_request_service import
|
20
|
+
from .join_request_service import join_request_from_flow
|
21
21
|
|
22
22
|
AGENT_STATUSES = {
|
23
23
|
'REQUESTS_MODIFIED': 'requests-modified'
|
@@ -50,7 +50,7 @@ def upload_request(
|
|
50
50
|
):
|
51
51
|
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Recording{bcolors.ENDC} {flow.request.url}")
|
52
52
|
|
53
|
-
joined_request =
|
53
|
+
joined_request = join_request_from_flow(flow, intercept_settings)
|
54
54
|
|
55
55
|
project_key = intercept_settings.project_key
|
56
56
|
scenario_key = intercept_settings.scenario_key
|
@@ -32,6 +32,7 @@ class ReplayRequestOptions(TypedDict):
|
|
32
32
|
report_key: Union[str, None]
|
33
33
|
request_origin: Union[request_origin.CLI, None]
|
34
34
|
response_fixtures_path: str
|
35
|
+
response_mode: Union[mode.RECORD, None]
|
35
36
|
scenario_key: Union[str, None]
|
36
37
|
scheme: str
|
37
38
|
test_filter: test_filter.TestFilter
|
@@ -83,6 +84,9 @@ def replay(context: ReplayContext, options: ReplayRequestOptions) -> requests.Re
|
|
83
84
|
if options.get('response_fixtures_path'):
|
84
85
|
__handle_path_header(custom_headers.RESPONSE_FIXTURES_PATH, options['response_fixtures_path'], headers)
|
85
86
|
|
87
|
+
if options.get('response_mode'):
|
88
|
+
headers[custom_headers.RESPONSE_PROXY_MODE] = options['response_mode']
|
89
|
+
|
86
90
|
if options.get('scenario_key'):
|
87
91
|
headers[custom_headers.SCENARIO_KEY] = options['scenario_key']
|
88
92
|
|
@@ -173,10 +177,6 @@ def __handle_mode_option(_mode, request: Request, headers):
|
|
173
177
|
headers[custom_headers.MOCK_REQUEST_ID] = str(request.id)
|
174
178
|
|
175
179
|
headers[custom_headers.MOCK_POLICY] = mock_policy.ALL
|
176
|
-
elif _mode == mode.RECORD:
|
177
|
-
# If recording, then it's actually a replay and record
|
178
|
-
headers[custom_headers.PROXY_MODE] = mode.REPLAY
|
179
|
-
headers[custom_headers.RESPONSE_PROXY_MODE] = mode.RECORD
|
180
180
|
|
181
181
|
def __create_replayed_response(request_id: int, res: requests.Response, latency: int):
|
182
182
|
replayed_response_model = ReplayedResponseModel(Settings.instance())
|
@@ -11,7 +11,7 @@ from stoobly_agent.lib.logger import Logger, bcolors
|
|
11
11
|
from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
|
12
12
|
|
13
13
|
from ...intercept_settings import InterceptSettings
|
14
|
-
from ...record.join_request_service import
|
14
|
+
from ...record.join_request_service import join_request_from_flow
|
15
15
|
|
16
16
|
LOG_ID = 'Test'
|
17
17
|
|
@@ -43,7 +43,7 @@ def upload_test(
|
|
43
43
|
flow: MitmproxyHTTPFlow,
|
44
44
|
**kwargs: UploadTestData
|
45
45
|
) -> Response:
|
46
|
-
joined_request =
|
46
|
+
joined_request = join_request_from_flow(flow, intercept_settings)
|
47
47
|
|
48
48
|
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Uploading{bcolors.ENDC} test results for {joined_request.proxy_request.url()}")
|
49
49
|
|
@@ -6,7 +6,7 @@ from typing import List
|
|
6
6
|
|
7
7
|
from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
|
8
8
|
from stoobly_agent.app.settings.firewall_rule import FirewallRule
|
9
|
-
from stoobly_agent.config.constants import
|
9
|
+
from stoobly_agent.config.constants import intercept_policy, request_origin
|
10
10
|
from stoobly_agent.lib.logger import bcolors, Logger
|
11
11
|
|
12
12
|
LOG_ID = 'Firewall'
|
@@ -18,8 +18,8 @@ def get_active_mode_policy(request: MitmproxyRequest, intercept_settings: Interc
|
|
18
18
|
if allowed_request(request, intercept_settings):
|
19
19
|
return intercept_settings.policy
|
20
20
|
else:
|
21
|
-
# If the request path does not match accepted paths, do not
|
22
|
-
return
|
21
|
+
# If the request path does not match accepted paths, do not intercept
|
22
|
+
return intercept_policy.NONE
|
23
23
|
|
24
24
|
def allowed_request(request: MitmproxyRequest, intercept_settings: InterceptSettings) -> bool:
|
25
25
|
# If an exclude rule(s) exists, then only requests not matching these pattern(s) are allowed
|
@@ -36,8 +36,6 @@ def bad_request(flow: MitmproxyHTTPFlow, message: str):
|
|
36
36
|
{'Content-Type': 'text/plain'} # (optional) headers
|
37
37
|
)
|
38
38
|
|
39
|
-
return False
|
40
|
-
|
41
39
|
# Without deleting this header, causes parsing issues when reading response
|
42
40
|
def disable_transfer_encoding(response: MitmproxyResponse) -> None:
|
43
41
|
header_name = 'Transfer-Encoding'
|
@@ -0,0 +1,72 @@
|
|
1
|
+
from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
|
2
|
+
from typing import List
|
3
|
+
|
4
|
+
from stoobly_agent.app.settings.constants.request_component import REQUEST_COMPONENTS, RESPONSE_COMPONENTS
|
5
|
+
from stoobly_agent.app.settings.rewrite_rule import RewriteRule
|
6
|
+
|
7
|
+
from ..mitmproxy.request_facade import MitmproxyRequestFacade
|
8
|
+
from ..mitmproxy.response_facade import MitmproxyResponseFacade
|
9
|
+
|
10
|
+
def select_request_rewrite_rules(rewrite_rules: List[RewriteRule]):
|
11
|
+
rules = []
|
12
|
+
|
13
|
+
for rewrite_rule in rewrite_rules:
|
14
|
+
parameter_rules = list(filter(
|
15
|
+
lambda parameter: parameter.type in REQUEST_COMPONENTS and parameter.name,
|
16
|
+
rewrite_rule.parameter_rules or []
|
17
|
+
))
|
18
|
+
|
19
|
+
if len(parameter_rules) > 0:
|
20
|
+
rewrite_rule = RewriteRule(rewrite_rule.to_dict())
|
21
|
+
rewrite_rule.url_rules = rewrite_rule.url_rules
|
22
|
+
rewrite_rule.parameter_rules = parameter_rules
|
23
|
+
rules.append(rewrite_rule)
|
24
|
+
|
25
|
+
return rules
|
26
|
+
|
27
|
+
def select_response_rewrite_rules(rewrite_rules: List[RewriteRule]):
|
28
|
+
rules = []
|
29
|
+
|
30
|
+
for rewrite_rule in rewrite_rules:
|
31
|
+
parameter_rules = list(filter(
|
32
|
+
lambda parameter: parameter.type in RESPONSE_COMPONENTS and parameter.name,
|
33
|
+
rewrite_rule.parameter_rules or []
|
34
|
+
))
|
35
|
+
|
36
|
+
if len(parameter_rules) > 0:
|
37
|
+
rewrite_rule = RewriteRule(rewrite_rule.to_dict())
|
38
|
+
rewrite_rule.url_rules = []
|
39
|
+
rewrite_rule.parameter_rules = parameter_rules
|
40
|
+
rules.append(rewrite_rule)
|
41
|
+
|
42
|
+
return rules
|
43
|
+
|
44
|
+
def rewrite_request_response(flow: MitmproxyHTTPFlow, rewrite_rules: List[RewriteRule]):
|
45
|
+
request = rewrite_request(flow, rewrite_rules)
|
46
|
+
response = rewrite_response(flow, rewrite_rules, request)
|
47
|
+
return request, response
|
48
|
+
|
49
|
+
def rewrite_request(flow: MitmproxyHTTPFlow, rewrite_rules: List[RewriteRule]):
|
50
|
+
request = None
|
51
|
+
|
52
|
+
# Adapt flow.request
|
53
|
+
request = MitmproxyRequestFacade(flow.request)
|
54
|
+
|
55
|
+
_rewrite_rules = select_request_rewrite_rules(rewrite_rules)
|
56
|
+
if len(_rewrite_rules):
|
57
|
+
request.with_parameter_rules(_rewrite_rules).with_url_rules(_rewrite_rules).rewrite()
|
58
|
+
|
59
|
+
return request
|
60
|
+
|
61
|
+
def rewrite_response(flow: MitmproxyHTTPFlow, rewrite_rules: List[RewriteRule], request = None):
|
62
|
+
# Adapt flow.request
|
63
|
+
request = request or MitmproxyRequestFacade(flow.request)
|
64
|
+
|
65
|
+
# Adapt flow.response
|
66
|
+
response = MitmproxyResponseFacade(flow.response)
|
67
|
+
|
68
|
+
_rewrite_rules = select_response_rewrite_rules(rewrite_rules)
|
69
|
+
if len(_rewrite_rules):
|
70
|
+
response.with_parameter_rules(_rewrite_rules, request).rewrite()
|
71
|
+
|
72
|
+
return response
|
@@ -6,4 +6,7 @@ QUERY_PARAM = 'Query Param'
|
|
6
6
|
RESPONSE_HEADER = 'Response Header'
|
7
7
|
RESPONSE_PARAM = 'Response Param'
|
8
8
|
|
9
|
-
|
9
|
+
REQUEST_COMPONENTS = [BODY_PARAM, HEADER, QUERY_PARAM]
|
10
|
+
RESPONSE_COMPONENTS = [RESPONSE_HEADER, RESPONSE_PARAM]
|
11
|
+
|
12
|
+
RequestComponent = Literal[BODY_PARAM, HEADER, QUERY_PARAM, RESPONSE_HEADER, RESPONSE_PARAM]
|