stoobly-agent 1.4.2__py3-none-any.whl → 1.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. stoobly_agent/__init__.py +1 -1
  2. stoobly_agent/app/cli/helpers/handle_mock_service.py +6 -2
  3. stoobly_agent/app/cli/helpers/request_facade.py +5 -1
  4. stoobly_agent/app/cli/scaffold/constants.py +1 -1
  5. stoobly_agent/app/cli/scaffold/docker/workflow/mock_decorator.py +1 -0
  6. stoobly_agent/app/cli/scaffold/service_workflow_validate_command.py +19 -19
  7. stoobly_agent/app/cli/scaffold/templates/app/.Dockerfile.context +1 -1
  8. stoobly_agent/app/cli/scaffold/templates/constants.py +3 -3
  9. stoobly_agent/app/cli/scaffold/templates/factory.py +5 -5
  10. stoobly_agent/app/cli/scaffold/templates/workflow/mock/bin/configure +1 -8
  11. stoobly_agent/app/cli/scaffold/templates/workflow/mock/fixtures.yml +1 -1
  12. stoobly_agent/app/cli/scaffold/templates/workflow/test/bin/configure +1 -8
  13. stoobly_agent/app/cli/scaffold/templates/workflow/test/fixtures.yml +1 -1
  14. stoobly_agent/app/cli/scaffold/workflow_command.py +3 -3
  15. stoobly_agent/app/cli/scaffold/workflow_create_command.py +2 -2
  16. stoobly_agent/app/cli/scaffold_cli.py +5 -5
  17. stoobly_agent/app/proxy/context.py +4 -0
  18. stoobly_agent/app/proxy/handle_mock_service.py +81 -54
  19. stoobly_agent/app/proxy/handle_record_service.py +15 -3
  20. stoobly_agent/app/proxy/handle_replay_service.py +44 -18
  21. stoobly_agent/app/proxy/handle_test_service.py +75 -16
  22. stoobly_agent/app/proxy/intercept_handler.py +11 -16
  23. stoobly_agent/app/proxy/intercept_settings.py +17 -4
  24. stoobly_agent/app/proxy/mitmproxy/request_facade.py +5 -2
  25. stoobly_agent/app/proxy/mitmproxy/response_facade.py +5 -4
  26. stoobly_agent/app/proxy/mock/eval_fixtures_service.py +78 -14
  27. stoobly_agent/app/proxy/mock/eval_request_service.py +2 -2
  28. stoobly_agent/app/proxy/record/join_request_service.py +7 -8
  29. stoobly_agent/app/proxy/record/upload_request_service.py +2 -2
  30. stoobly_agent/app/proxy/replay/replay_request_service.py +4 -4
  31. stoobly_agent/app/proxy/test/helpers/upload_test_service.py +2 -2
  32. stoobly_agent/app/proxy/utils/allowed_request_service.py +3 -3
  33. stoobly_agent/app/proxy/utils/response_handler.py +0 -2
  34. stoobly_agent/app/proxy/utils/rewrite.py +72 -0
  35. stoobly_agent/app/settings/constants/request_component.py +4 -1
  36. stoobly_agent/cli.py +35 -28
  37. stoobly_agent/config/constants/intercept_policy.py +2 -0
  38. stoobly_agent/config/constants/mock_policy.py +4 -2
  39. stoobly_agent/config/constants/record_policy.py +4 -2
  40. stoobly_agent/config/constants/replay_policy.py +4 -2
  41. stoobly_agent/public/{18-es2015.583f191cc7ad512ee262.js → 18-es2015.503207073756a9c8211a.js} +1 -1
  42. stoobly_agent/public/{18-es5.583f191cc7ad512ee262.js → 18-es5.503207073756a9c8211a.js} +1 -1
  43. stoobly_agent/public/index.html +1 -1
  44. stoobly_agent/public/{main-es2015.2cc16523aa3fcaba51e5.js → main-es2015.d682619f3d6d53d64c6a.js} +1 -1
  45. stoobly_agent/public/{main-es5.2cc16523aa3fcaba51e5.js → main-es5.d682619f3d6d53d64c6a.js} +1 -1
  46. stoobly_agent/public/{runtime-es2015.b914470164e4d6e75d96.js → runtime-es2015.8c1efed946fc02c923fc.js} +1 -1
  47. stoobly_agent/public/{runtime-es5.b914470164e4d6e75d96.js → runtime-es5.8c1efed946fc02c923fc.js} +1 -1
  48. stoobly_agent/test/app/cli/helpers/openapi_endpoint_adapter_test.py +2 -1
  49. stoobly_agent/test/app/cli/scaffold/e2e_test.py +2 -2
  50. stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION +1 -1
  51. stoobly_agent/test/app/proxy/mock/eval_fixtures_service_test.py +140 -71
  52. stoobly_agent/test/cli/lifecycle_hooks_test.py +66 -0
  53. stoobly_agent/test/cli/mock_test.py +53 -29
  54. stoobly_agent/test/cli/record_test.py +67 -0
  55. stoobly_agent/test/mock_data/lifecycle_hooks.py +35 -0
  56. {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/LICENSE +1 -1
  57. {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/METADATA +7 -12
  58. {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/RECORD +62 -58
  59. /stoobly_agent/app/cli/scaffold/templates/workflow/mock/{fixtures/.keep → public/.gitignore} +0 -0
  60. /stoobly_agent/app/cli/scaffold/templates/workflow/test/{fixtures/.keep → public/.gitignore} +0 -0
  61. {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/WHEEL +0 -0
  62. {stoobly_agent-1.4.2.dist-info → stoobly_agent-1.5.0.dist-info}/entry_points.txt +0 -0
@@ -1,7 +1,9 @@
1
1
  import pdb
2
2
 
3
+ from copy import deepcopy
3
4
  from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
4
5
 
6
+ from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
5
7
  from stoobly_agent.app.proxy.replay.body_parser_service import encode_response
6
8
  from stoobly_agent.app.proxy.replay.context import ReplayContext
7
9
  from stoobly_agent.app.proxy.utils.request_handler import build_response
@@ -12,35 +14,53 @@ from stoobly_agent.lib.api.endpoints_resource import EndpointsResource
12
14
  from stoobly_agent.lib.api.interfaces.tests import TestShowResponse
13
15
  from stoobly_agent.lib.logger import Logger
14
16
 
15
- from .handle_mock_service import handle_request_mock_generic
16
- from .handle_replay_service import handle_request_replay
17
+ from .handle_mock_service import handle_request_mock_generic_without_rewrite
18
+ from .handle_replay_service import handle_request_replay_without_rewrite
17
19
  from .mock.context import MockContext
18
20
  from .test.helpers.test_results_builder import TestResultsBuilder
19
21
  from .test.helpers.upload_test_service import inject_upload_test
20
22
  from .test.context_abc import TestContextABC as TestContext
21
23
  from .test.test_service import test
24
+ from .utils.rewrite import rewrite_request, rewrite_response, rewrite_request_response
22
25
 
23
26
  LOG_ID = 'HandleTest'
24
27
 
28
+ ###
29
+ #
30
+ # 1. Rewrites test request
31
+ # 2. BEFORE_REPLAY gets triggered
32
+ #
25
33
  def handle_request_test(context: ReplayContext) -> None:
26
- handle_request_replay(context)
34
+ __rewrite_request(context)
35
+ handle_request_replay_without_rewrite(context)
27
36
 
28
37
  ###
29
38
  #
30
- # Mock and Test modes share the same policies
39
+ # 1. Rewrites test response (response from live service)
40
+ # 2. AFTER_REPLAY gets triggered
41
+ # 3. Uses rewritten test request to obtain mock response
42
+ # 4. BEFORE_MOCK gets triggered
43
+ # 5. AFTER_MOCK gets triggered
44
+ # 6. BEFORE_TEST gets triggered
45
+ # 7. Tests against rewritten test response and mock response (expected response)
46
+ # 8. Rewrites a copy of request and response
47
+ # 9. BEFORE_RECORD gets triggered (if not from CLI)
48
+ # 10. AFTER_RECORD gets triggered (if not from CLI)
49
+ # 11. AFTER_TEST gets triggered
31
50
  #
32
51
  def handle_response_test(context: ReplayContext) -> None:
33
52
  from .test.context import TestContext
34
53
 
35
54
  flow: MitmproxyHTTPFlow = context.flow
36
- intercept_settings = context.intercept_settings
37
-
38
55
  disable_transfer_encoding(flow.response)
39
56
 
40
- # At this point, the request may already been rewritten during replay
41
- # Request will be rewritten again for mocking purposes
57
+ __rewrite_response(context)
58
+ __test_hook(lifecycle_hooks.AFTER_REPLAY, context)
42
59
 
43
- handle_request_mock_generic(
60
+ intercept_settings = context.intercept_settings
61
+
62
+ # At this point, the request may already been rewritten during replay, do not rewrite again
63
+ handle_request_mock_generic_without_rewrite(
44
64
  MockContext(flow, intercept_settings),
45
65
  failure=lambda mock_context: __handle_mock_failure(TestContext(context, mock_context)),
46
66
  #infer=intercept_settings.test_strategy == test_strategy.FUZZY, # For fuzzy testing we can use an inferred response
@@ -53,8 +73,10 @@ def __decorate_test_id(flow: MitmproxyHTTPFlow, test_response: TestShowResponse)
53
73
 
54
74
  def __handle_mock_success(test_context: TestContext) -> None:
55
75
  flow: MitmproxyHTTPFlow = test_context.flow
56
- settings = Settings.instance()
57
76
 
77
+ __test_hook(lifecycle_hooks.AFTER_MOCK, test_context.mock_context)
78
+
79
+ settings: Settings = Settings.instance()
58
80
  test_context.with_endpoints_resource(EndpointsResource(settings.remote.api_url, settings.remote.api_key))
59
81
 
60
82
  __test_hook(lifecycle_hooks.BEFORE_TEST, test_context)
@@ -84,13 +106,8 @@ def __handle_mock_success(test_context: TestContext) -> None:
84
106
  if not is_cli or test_context.save:
85
107
  # Re-serialize expected response since it was rewritten
86
108
  upload_test_data['expected_response'] = encode_response(expected, test_context.expected_response.content_type)
87
- upload_test = inject_upload_test(None, intercept_settings)
88
109
 
89
- # Commit test to API
90
- res = upload_test(
91
- flow,
92
- **upload_test_data
93
- )
110
+ res = __record_handler(test_context, upload_test_data)
94
111
 
95
112
  if is_cli:
96
113
  # If the origin was from a CLI, send test ID in response header
@@ -136,6 +153,48 @@ def __override_response(flow: MitmproxyHTTPFlow, content: bytes):
136
153
  flow.response.set_content(content)
137
154
  flow.response.status_code = 200
138
155
 
156
+ def __record_handler(context: TestContext, upload_test_data):
157
+ flow = context.flow
158
+ flow_copy = deepcopy(flow) # Deep copy flow to prevent response modifications from persisting
159
+ intercept_settings = context.intercept_settings
160
+
161
+ context.flow = flow_copy
162
+
163
+ # Since we are "uploading" the request, use record_write_rules
164
+ rewrite_request_response(flow_copy, intercept_settings.record_rewrite_rules)
165
+ __test_hook(lifecycle_hooks.BEFORE_RECORD, context)
166
+
167
+ # Commit test to API
168
+ upload_test = inject_upload_test(None, intercept_settings)
169
+ res = upload_test(
170
+ flow_copy, **upload_test_data
171
+ )
172
+
173
+ __test_hook(lifecycle_hooks.AFTER_RECORD, context)
174
+ context.flow = flow
175
+
176
+ return res
177
+
178
+ def __rewrite_request(replay_context: ReplayContext):
179
+ """
180
+ Before replaying a request, see if the request needs to be rewritten
181
+ """
182
+ intercept_settings: InterceptSettings = replay_context.intercept_settings
183
+ rewrite_rules = intercept_settings.test_rewrite_rules
184
+
185
+ if len(rewrite_rules) > 0:
186
+ rewrite_request(replay_context.flow, rewrite_rules)
187
+
188
+ def __rewrite_response(replay_context: ReplayContext):
189
+ """
190
+ After replaying a request, see if the request needs to be rewritten
191
+ """
192
+ intercept_settings: InterceptSettings = replay_context.intercept_settings
193
+ rewrite_rules = intercept_settings.test_rewrite_rules
194
+
195
+ if len(rewrite_rules) > 0:
196
+ rewrite_response(replay_context.flow, rewrite_rules)
197
+
139
198
  def __test_hook(hook: str, context: TestContext):
140
199
  intercept_settings = context.intercept_settings
141
200
  lifecycle_hooks_module = intercept_settings.lifecycle_hooks
@@ -33,6 +33,7 @@ def request(flow: MitmproxyHTTPFlow):
33
33
  if not intercept_settings.active:
34
34
  return
35
35
 
36
+ __disable_web_cache(request)
36
37
  __intercept_hook(lifecycle_hooks.BEFORE_REQUEST, flow, intercept_settings)
37
38
 
38
39
  active_mode = intercept_settings.mode
@@ -42,7 +43,7 @@ def request(flow: MitmproxyHTTPFlow):
42
43
  context = MockContext(flow, intercept_settings)
43
44
  handle_request_mock(context)
44
45
  elif active_mode == mode.RECORD:
45
- __disable_web_cache(request)
46
+ pass
46
47
  elif active_mode == mode.REPLAY:
47
48
  context = ReplayContext(flow, intercept_settings)
48
49
  handle_request_replay(context)
@@ -65,35 +66,29 @@ def response(flow: MitmproxyHTTPFlow):
65
66
  if not intercept_settings.active:
66
67
  return
67
68
 
68
- __intercept_hook(lifecycle_hooks.BEFORE_RESPONSE, flow, intercept_settings)
69
-
70
69
  active_mode = intercept_settings.mode
71
70
 
72
71
  if active_mode == mode.MOCK:
73
72
  context = MockContext(flow, intercept_settings)
74
- return handle_response_mock(context)
73
+ handle_response_mock(context)
75
74
  elif active_mode == mode.RECORD:
76
75
  context = RecordContext(flow, intercept_settings)
77
- return handle_response_record(context)
76
+ handle_response_record(context)
78
77
  elif active_mode == mode.REPLAY:
79
78
  context = ReplayContext(flow, intercept_settings)
80
- return handle_response_replay(context)
79
+ handle_response_replay(context)
81
80
  elif active_mode == mode.TEST:
82
81
  context = ReplayContext(flow, intercept_settings)
83
- return handle_response_test(context)
82
+ handle_response_test(context)
83
+
84
+ __intercept_hook(lifecycle_hooks.BEFORE_RESPONSE, flow, intercept_settings)
84
85
 
85
86
  ### PRIVATE
86
87
 
88
+ # Prevent 304 status
89
+ # Because this header will get recorded, should add during mocking as well in the case where headers are used for matching
87
90
  def __disable_web_cache(request: MitmproxyRequest) -> None:
88
- request.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
89
- request.headers['Expires'] = '0'
90
- request.headers['Pragma'] = 'no-cache'
91
-
92
- if 'IF-NONE-MATCH' in request.headers:
93
- del request.headers['IF-NONE-MATCH']
94
-
95
- if 'IF-MODIFIED-SINCE' in request.headers:
96
- del request.headers['IF-MODIFIED-SINCE']
91
+ request.headers['Cache-Control'] = 'no-cache, no-store'
97
92
 
98
93
  # Fix issue where multi-value cookies become comma separated
99
94
  def __patch_cookie(request: MitmproxyRequest):
@@ -41,6 +41,11 @@ class InterceptSettings:
41
41
  self.__response_fixtures = None
42
42
  self.__initialize_response_fixtures()
43
43
 
44
+ self._mock_rewrite_rules = None
45
+ self._record_rewrite_rules = None
46
+ self._replay_rewrite_rules = None
47
+ self._test_rewrite_rules = None
48
+
44
49
  @property
45
50
  def settings(self):
46
51
  return self.__settings
@@ -200,19 +205,27 @@ class InterceptSettings:
200
205
 
201
206
  @property
202
207
  def record_rewrite_rules(self) -> List[RewriteRule]:
203
- return self.__select_rewrite_rules(mode.RECORD)
208
+ if not self._record_rewrite_rules:
209
+ self._record_rewrite_rules = self.__select_rewrite_rules(mode.RECORD)
210
+ return self._record_rewrite_rules
204
211
 
205
212
  @property
206
213
  def mock_rewrite_rules(self) -> List[RewriteRule]:
207
- return self.__select_rewrite_rules(mode.MOCK)
214
+ if not self._mock_rewrite_rules:
215
+ self._mock_rewrite_rules = self.__select_rewrite_rules(mode.MOCK)
216
+ return self._mock_rewrite_rules
208
217
 
209
218
  @property
210
219
  def replay_rewrite_rules(self) -> List[RewriteRule]:
211
- return self.__select_rewrite_rules(mode.REPLAY)
220
+ if not self._replay_rewrite_rules:
221
+ self._replay_rewrite_rules = self.__select_rewrite_rules(mode.REPLAY)
222
+ return self._replay_rewrite_rules
212
223
 
213
224
  @property
214
225
  def test_rewrite_rules(self) -> List[RewriteRule]:
215
- return self.__select_rewrite_rules(mode.TEST)
226
+ if not self._test_rewrite_rules:
227
+ self._test_rewrite_rules = self.__select_rewrite_rules(mode.TEST)
228
+ return self._test_rewrite_rules
216
229
 
217
230
  @property
218
231
  def upstream_url(self):
@@ -239,7 +239,7 @@ class MitmproxyRequestFacade(Request):
239
239
  content_type = self.content_type
240
240
  parsed_content = self.__body.get(content_type)
241
241
 
242
- if not isinstance(parsed_content, dict) and not isinstance(parsed_content, multidict.MultiDictView):
242
+ if not self.__is_iterable(parsed_content):
243
243
  content_type = 'application/json'
244
244
  self.request.headers['content-type'] = content_type
245
245
  parsed_content = {}
@@ -265,4 +265,7 @@ class MitmproxyRequestFacade(Request):
265
265
 
266
266
  _request_headers.pop(name)
267
267
 
268
- return _request_headers
268
+ return _request_headers
269
+
270
+ def __is_iterable(self, v):
271
+ return isinstance(v, dict) or isinstance(v, multidict.MultiDictView) or isinstance(v, list)
@@ -95,10 +95,8 @@ class MitmproxyResponseFacade(Response):
95
95
  content_type = self.content_type
96
96
  parsed_content = self.__body.get(content_type)
97
97
 
98
- if not isinstance(parsed_content, dict) and not isinstance(parsed_content, multidict.MultiDictView):
99
- content_type = 'application/json'
100
- self.response.headers['content-type'] = content_type
101
- parsed_content = {}
98
+ if not self.__is_iterable(parsed_content):
99
+ return
102
100
 
103
101
  self.__apply_rewrites(parsed_content, rewrites, handler)
104
102
  self.__body.set(parsed_content, content_type)
@@ -122,3 +120,6 @@ class MitmproxyResponseFacade(Response):
122
120
  _response_headers.pop(name)
123
121
 
124
122
  return _response_headers
123
+
124
+ def __is_iterable(self, v):
125
+ return isinstance(v, dict) or isinstance(v, multidict.MultiDictView) or isinstance(v, list)
@@ -1,3 +1,4 @@
1
+ import mimetypes
1
2
  import os
2
3
  import pdb
3
4
  import re
@@ -5,6 +6,7 @@ import re
5
6
  from io import BytesIO
6
7
  from mitmproxy.http import Request as MitmproxyRequest
7
8
  from requests import Response
9
+ from requests.structures import CaseInsensitiveDict
8
10
  from typing import Union
9
11
 
10
12
  from stoobly_agent.lib.logger import bcolors, Logger
@@ -19,45 +21,88 @@ class Options():
19
21
 
20
22
  def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Response, None]:
21
23
  fixture_path = None
22
- headers = {}
24
+ headers = CaseInsensitiveDict()
25
+ status_code = 200
23
26
 
24
27
  response_fixtures = options.get('response_fixtures')
25
- fixture = __eval_response_fixtures(request, response_fixtures)
28
+ fixture: dict = __eval_response_fixtures(request, response_fixtures)
26
29
 
27
30
  if not fixture:
28
31
  public_directory_path = options.get('public_directory_path')
29
32
 
30
- if public_directory_path and os.path.exists(public_directory_path):
31
- static_file_path = os.path.join(public_directory_path, request.path.lstrip('/'))
33
+ if not public_directory_path:
34
+ return
32
35
 
33
- if os.path.exists(static_file_path):
34
- fixture_path = static_file_path
36
+ request_path = 'index' if request.path == '/' else request.path
37
+ _fixture_path = os.path.join(public_directory_path, request_path.lstrip('/'))
38
+ if request.headers.get('accept'):
39
+ fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
40
+
41
+ if not fixture_path:
42
+ fixture_path = _fixture_path
43
+ if not os.path.isfile(fixture_path):
44
+ return
35
45
  else:
36
46
  fixture_path = fixture.get('path')
37
- headers = fixture.get('headers') or {}
47
+ if not fixture_path or not os.path.isfile(fixture_path):
48
+ return
38
49
 
39
- if not fixture_path:
40
- return
50
+ _headers = fixture.get('headers')
51
+ headers = CaseInsensitiveDict(_headers if isinstance(_headers, dict) else {})
52
+
53
+ if fixture.get('status_code'):
54
+ status_code = fixture.get('status_code')
41
55
 
42
56
  with open(fixture_path, 'rb') as fp:
43
57
  response = Response()
44
58
 
45
- response.status_code = 200
59
+ response.status_code = status_code
46
60
  response.raw = BytesIO(fp.read())
47
61
  response.headers = headers
48
62
 
63
+ if not response.headers.get('content-type'):
64
+ content_type = __guess_content_type(fixture_path)
65
+ if content_type:
66
+ response.headers['content-type'] = content_type
67
+
49
68
  Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE}Resolved{bcolors.ENDC} fixture {fixture_path}")
50
69
 
51
70
  return response
52
71
 
72
+ def __guess_content_type(file_path):
73
+ file_extension = os.path.splitext(file_path)[1]
74
+ if not file_extension:
75
+ return
76
+ return mimetypes.types_map.get(file_extension)
77
+
78
+ def __guess_file_path(file_path, content_type):
79
+ file_extension = os.path.splitext(file_path)[1]
80
+ if file_extension:
81
+ return file_path
82
+
83
+ if not content_type:
84
+ return
85
+
86
+ content_types = __parse_accept_header(content_type)
87
+
88
+ for content_type in content_types:
89
+ file_extension = mimetypes.guess_extension(content_type)
90
+
91
+ if not file_extension:
92
+ continue
93
+
94
+ _file_path = f"{file_path}{file_extension}"
95
+ if os.path.isfile(_file_path):
96
+ return _file_path
97
+
53
98
  def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtures):
54
- if not response_fixtures:
99
+ if not isinstance(response_fixtures, dict):
55
100
  return
56
101
 
57
102
  method = request.method
58
103
  routes = response_fixtures.get(method)
59
104
 
60
- if not routes:
105
+ if not isinstance(routes, dict):
61
106
  return
62
107
 
63
108
  for path_pattern in routes:
@@ -65,7 +110,26 @@ def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtu
65
110
  continue
66
111
 
67
112
  fixture = routes[path_pattern]
113
+ if not isinstance(fixture, dict):
114
+ continue
115
+
68
116
  path = fixture.get('path')
69
117
 
70
- if path and os.path.exists(path):
71
- return fixture
118
+ if path:
119
+ return fixture
120
+
121
+ def __parse_accept_header(accept_header):
122
+ types = []
123
+ for item in accept_header.split(","):
124
+ parts = item.split(";")
125
+ content_type = parts[0].strip()
126
+ q_value = 1.0 # Default quality value
127
+ if len(parts) > 1 and parts[1].strip().startswith("q="):
128
+ try:
129
+ q_value = float(parts[1].strip()[2:])
130
+ except ValueError:
131
+ pass # Keep default q_value if parsing fails
132
+ types.append((content_type, q_value))
133
+
134
+ # Sort by quality factor in descending order
135
+ return [content_type for content_type, _ in sorted(types, key=lambda x: x[1], reverse=True)]
@@ -149,9 +149,9 @@ def __filter_by_match_rules(request: MitmproxyRequest, match_rules: List[MatchRu
149
149
  method = request.method.upper()
150
150
 
151
151
  keep = {
152
- request_component.BODY_PARAM: True,
152
+ request_component.BODY_PARAM: False,
153
153
  request_component.HEADER: False,
154
- request_component.QUERY_PARAM: True,
154
+ request_component.QUERY_PARAM: False,
155
155
  }
156
156
 
157
157
  match_rules = list(filter(lambda rule: method in rule.methods, match_rules))
@@ -1,10 +1,14 @@
1
1
  import pdb
2
+
2
3
  from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
4
+ from typing import List
3
5
 
4
6
  from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
7
+ from stoobly_agent.app.settings.rewrite_rule import RewriteRule
5
8
 
6
9
  from ..mitmproxy.request_facade import MitmproxyRequestFacade
7
10
  from ..mitmproxy.response_facade import MitmproxyResponseFacade
11
+ from ..utils.rewrite import rewrite_request_response
8
12
  from .joined_request import JoinedRequest
9
13
  from .proxy_request import ProxyRequest
10
14
 
@@ -18,15 +22,10 @@ def join_request(
18
22
  # Create JoinedRequest
19
23
  return JoinedRequest(proxy_request).with_response(adapted_response)
20
24
 
21
- def join_rewritten_request(flow: MitmproxyHTTPFlow, intercept_settings: InterceptSettings) -> JoinedRequest:
22
- # Adapt flow.request
25
+ def join_request_from_flow(
26
+ flow: MitmproxyHTTPFlow, intercept_settings: InterceptSettings
27
+ ) -> JoinedRequest:
23
28
  request = MitmproxyRequestFacade(flow.request)
24
-
25
- # Adapt flow.response
26
29
  response = MitmproxyResponseFacade(flow.response)
27
- rewrite_rules = intercept_settings.record_rewrite_rules
28
-
29
- request.with_parameter_rules(rewrite_rules).with_url_rules(rewrite_rules).rewrite()
30
- response.with_parameter_rules(rewrite_rules, request).rewrite()
31
30
 
32
31
  return join_request(request, response, intercept_settings)
@@ -17,7 +17,7 @@ from stoobly_agent.lib.logger import Logger, bcolors
17
17
  from stoobly_agent.lib.orm.request import Request
18
18
 
19
19
  from ..utils.publish_change_service import publish_requests_modified
20
- from .join_request_service import join_rewritten_request
20
+ from .join_request_service import join_request_from_flow
21
21
 
22
22
  AGENT_STATUSES = {
23
23
  'REQUESTS_MODIFIED': 'requests-modified'
@@ -50,7 +50,7 @@ def upload_request(
50
50
  ):
51
51
  Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Recording{bcolors.ENDC} {flow.request.url}")
52
52
 
53
- joined_request = join_rewritten_request(flow, intercept_settings)
53
+ joined_request = join_request_from_flow(flow, intercept_settings)
54
54
 
55
55
  project_key = intercept_settings.project_key
56
56
  scenario_key = intercept_settings.scenario_key
@@ -32,6 +32,7 @@ class ReplayRequestOptions(TypedDict):
32
32
  report_key: Union[str, None]
33
33
  request_origin: Union[request_origin.CLI, None]
34
34
  response_fixtures_path: str
35
+ response_mode: Union[mode.RECORD, None]
35
36
  scenario_key: Union[str, None]
36
37
  scheme: str
37
38
  test_filter: test_filter.TestFilter
@@ -83,6 +84,9 @@ def replay(context: ReplayContext, options: ReplayRequestOptions) -> requests.Re
83
84
  if options.get('response_fixtures_path'):
84
85
  __handle_path_header(custom_headers.RESPONSE_FIXTURES_PATH, options['response_fixtures_path'], headers)
85
86
 
87
+ if options.get('response_mode'):
88
+ headers[custom_headers.RESPONSE_PROXY_MODE] = options['response_mode']
89
+
86
90
  if options.get('scenario_key'):
87
91
  headers[custom_headers.SCENARIO_KEY] = options['scenario_key']
88
92
 
@@ -173,10 +177,6 @@ def __handle_mode_option(_mode, request: Request, headers):
173
177
  headers[custom_headers.MOCK_REQUEST_ID] = str(request.id)
174
178
 
175
179
  headers[custom_headers.MOCK_POLICY] = mock_policy.ALL
176
- elif _mode == mode.RECORD:
177
- # If recording, then it's actually a replay and record
178
- headers[custom_headers.PROXY_MODE] = mode.REPLAY
179
- headers[custom_headers.RESPONSE_PROXY_MODE] = mode.RECORD
180
180
 
181
181
  def __create_replayed_response(request_id: int, res: requests.Response, latency: int):
182
182
  replayed_response_model = ReplayedResponseModel(Settings.instance())
@@ -11,7 +11,7 @@ from stoobly_agent.lib.logger import Logger, bcolors
11
11
  from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
12
12
 
13
13
  from ...intercept_settings import InterceptSettings
14
- from ...record.join_request_service import join_rewritten_request
14
+ from ...record.join_request_service import join_request_from_flow
15
15
 
16
16
  LOG_ID = 'Test'
17
17
 
@@ -43,7 +43,7 @@ def upload_test(
43
43
  flow: MitmproxyHTTPFlow,
44
44
  **kwargs: UploadTestData
45
45
  ) -> Response:
46
- joined_request = join_rewritten_request(flow, intercept_settings)
46
+ joined_request = join_request_from_flow(flow, intercept_settings)
47
47
 
48
48
  Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Uploading{bcolors.ENDC} test results for {joined_request.proxy_request.url()}")
49
49
 
@@ -6,7 +6,7 @@ from typing import List
6
6
 
7
7
  from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
8
8
  from stoobly_agent.app.settings.firewall_rule import FirewallRule
9
- from stoobly_agent.config.constants import mock_policy, request_origin
9
+ from stoobly_agent.config.constants import intercept_policy, request_origin
10
10
  from stoobly_agent.lib.logger import bcolors, Logger
11
11
 
12
12
  LOG_ID = 'Firewall'
@@ -18,8 +18,8 @@ def get_active_mode_policy(request: MitmproxyRequest, intercept_settings: Interc
18
18
  if allowed_request(request, intercept_settings):
19
19
  return intercept_settings.policy
20
20
  else:
21
- # If the request path does not match accepted paths, do not mock
22
- return mock_policy.NONE
21
+ # If the request path does not match accepted paths, do not intercept
22
+ return intercept_policy.NONE
23
23
 
24
24
  def allowed_request(request: MitmproxyRequest, intercept_settings: InterceptSettings) -> bool:
25
25
  # If an exclude rule(s) exists, then only requests not matching these pattern(s) are allowed
@@ -36,8 +36,6 @@ def bad_request(flow: MitmproxyHTTPFlow, message: str):
36
36
  {'Content-Type': 'text/plain'} # (optional) headers
37
37
  )
38
38
 
39
- return False
40
-
41
39
  # Without deleting this header, causes parsing issues when reading response
42
40
  def disable_transfer_encoding(response: MitmproxyResponse) -> None:
43
41
  header_name = 'Transfer-Encoding'
@@ -0,0 +1,72 @@
1
+ from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
2
+ from typing import List
3
+
4
+ from stoobly_agent.app.settings.constants.request_component import REQUEST_COMPONENTS, RESPONSE_COMPONENTS
5
+ from stoobly_agent.app.settings.rewrite_rule import RewriteRule
6
+
7
+ from ..mitmproxy.request_facade import MitmproxyRequestFacade
8
+ from ..mitmproxy.response_facade import MitmproxyResponseFacade
9
+
10
+ def select_request_rewrite_rules(rewrite_rules: List[RewriteRule]):
11
+ rules = []
12
+
13
+ for rewrite_rule in rewrite_rules:
14
+ parameter_rules = list(filter(
15
+ lambda parameter: parameter.type in REQUEST_COMPONENTS and parameter.name,
16
+ rewrite_rule.parameter_rules or []
17
+ ))
18
+
19
+ if len(parameter_rules) > 0:
20
+ rewrite_rule = RewriteRule(rewrite_rule.to_dict())
21
+ rewrite_rule.url_rules = rewrite_rule.url_rules
22
+ rewrite_rule.parameter_rules = parameter_rules
23
+ rules.append(rewrite_rule)
24
+
25
+ return rules
26
+
27
+ def select_response_rewrite_rules(rewrite_rules: List[RewriteRule]):
28
+ rules = []
29
+
30
+ for rewrite_rule in rewrite_rules:
31
+ parameter_rules = list(filter(
32
+ lambda parameter: parameter.type in RESPONSE_COMPONENTS and parameter.name,
33
+ rewrite_rule.parameter_rules or []
34
+ ))
35
+
36
+ if len(parameter_rules) > 0:
37
+ rewrite_rule = RewriteRule(rewrite_rule.to_dict())
38
+ rewrite_rule.url_rules = []
39
+ rewrite_rule.parameter_rules = parameter_rules
40
+ rules.append(rewrite_rule)
41
+
42
+ return rules
43
+
44
+ def rewrite_request_response(flow: MitmproxyHTTPFlow, rewrite_rules: List[RewriteRule]):
45
+ request = rewrite_request(flow, rewrite_rules)
46
+ response = rewrite_response(flow, rewrite_rules, request)
47
+ return request, response
48
+
49
+ def rewrite_request(flow: MitmproxyHTTPFlow, rewrite_rules: List[RewriteRule]):
50
+ request = None
51
+
52
+ # Adapt flow.request
53
+ request = MitmproxyRequestFacade(flow.request)
54
+
55
+ _rewrite_rules = select_request_rewrite_rules(rewrite_rules)
56
+ if len(_rewrite_rules):
57
+ request.with_parameter_rules(_rewrite_rules).with_url_rules(_rewrite_rules).rewrite()
58
+
59
+ return request
60
+
61
+ def rewrite_response(flow: MitmproxyHTTPFlow, rewrite_rules: List[RewriteRule], request = None):
62
+ # Adapt flow.request
63
+ request = request or MitmproxyRequestFacade(flow.request)
64
+
65
+ # Adapt flow.response
66
+ response = MitmproxyResponseFacade(flow.response)
67
+
68
+ _rewrite_rules = select_response_rewrite_rules(rewrite_rules)
69
+ if len(_rewrite_rules):
70
+ response.with_parameter_rules(_rewrite_rules, request).rewrite()
71
+
72
+ return response
@@ -6,4 +6,7 @@ QUERY_PARAM = 'Query Param'
6
6
  RESPONSE_HEADER = 'Response Header'
7
7
  RESPONSE_PARAM = 'Response Param'
8
8
 
9
- RequestComponent = Literal[BODY_PARAM, HEADER, QUERY_PARAM, RESPONSE_HEADER, RESPONSE_PARAM]
9
+ REQUEST_COMPONENTS = [BODY_PARAM, HEADER, QUERY_PARAM]
10
+ RESPONSE_COMPONENTS = [RESPONSE_HEADER, RESPONSE_PARAM]
11
+
12
+ RequestComponent = Literal[BODY_PARAM, HEADER, QUERY_PARAM, RESPONSE_HEADER, RESPONSE_PARAM]