stoobly-agent 0.34.9__py3-none-any.whl → 0.34.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stoobly_agent/__init__.py +1 -1
- stoobly_agent/app/api/proxy_controller.py +12 -10
- stoobly_agent/app/cli/config_cli.py +5 -3
- stoobly_agent/app/cli/helpers/handle_config_update_service.py +5 -0
- stoobly_agent/app/cli/snapshot_cli.py +85 -6
- stoobly_agent/app/models/adapters/python/response/mitmproxy_adapter.py +1 -1
- stoobly_agent/app/models/factories/resource/local_db/helpers/log.py +9 -7
- stoobly_agent/app/models/factories/resource/local_db/helpers/request_snapshot.py +5 -0
- stoobly_agent/app/models/factories/resource/local_db/helpers/scenario_snapshot.py +24 -0
- stoobly_agent/app/models/factories/resource/local_db/helpers/snapshot.py +18 -2
- stoobly_agent/app/models/factories/resource/local_db/helpers/tiebreak_scenario_request.py +6 -1
- stoobly_agent/app/models/factories/resource/local_db/query_param_adapter.py +9 -5
- stoobly_agent/app/models/factories/resource/local_db/request_adapter.py +1 -0
- stoobly_agent/app/proxy/handle_mock_service.py +16 -9
- stoobly_agent/app/proxy/handle_record_service.py +2 -2
- stoobly_agent/app/proxy/intercept_handler.py +7 -4
- stoobly_agent/app/proxy/mitmproxy/request_facade.py +4 -2
- stoobly_agent/app/proxy/mock/eval_fixtures_service.py +3 -1
- stoobly_agent/app/proxy/mock/hashed_request_decorator.py +10 -10
- stoobly_agent/app/proxy/mock/request_hasher.py +3 -1
- stoobly_agent/app/proxy/record/upload_request_service.py +5 -5
- stoobly_agent/app/proxy/replay/alias_resolver.py +5 -3
- stoobly_agent/app/proxy/replay/body_parser_service.py +1 -5
- stoobly_agent/app/proxy/replay/multipart.py +9 -27
- stoobly_agent/app/proxy/replay/trace_context.py +10 -9
- stoobly_agent/app/proxy/test/helpers/upload_test_service.py +5 -3
- stoobly_agent/app/proxy/utils/allowed_request_service.py +7 -5
- stoobly_agent/app/proxy/utils/request_handler.py +3 -1
- stoobly_agent/app/settings/__init__.py +32 -14
- stoobly_agent/cli.py +1 -1
- stoobly_agent/config/constants/custom_headers.py +1 -0
- stoobly_agent/config/data_dir.py +39 -21
- stoobly_agent/lib/api/body_param_names_resource.py +5 -3
- stoobly_agent/lib/api/endpoints_resource.py +5 -3
- stoobly_agent/lib/api/header_names_resource.py +5 -3
- stoobly_agent/lib/api/projects_resource.py +5 -3
- stoobly_agent/lib/api/query_param_names_resource.py +5 -3
- stoobly_agent/lib/api/requests_resource.py +5 -3
- stoobly_agent/lib/api/response_header_names_resource.py +5 -3
- stoobly_agent/lib/api/response_param_names_resource.py +5 -3
- stoobly_agent/lib/api/scenarios_resource.py +5 -3
- stoobly_agent/lib/api/stoobly_api.py +0 -1
- stoobly_agent/lib/api/test_responses_resource.py +3 -1
- stoobly_agent/lib/api/tests_resource.py +3 -1
- stoobly_agent/lib/api/users_resource.py +3 -1
- stoobly_agent/lib/cache.py +26 -9
- stoobly_agent/lib/logger.py +5 -2
- stoobly_agent/lib/utils/visitor.py +4 -3
- stoobly_agent/public/{13-es2015.220b4a1adf4cacb294e5.js → 13-es2015.343b0261a8b3b3f4a1fc.js} +1 -1
- stoobly_agent/public/{13-es5.220b4a1adf4cacb294e5.js → 13-es5.343b0261a8b3b3f4a1fc.js} +1 -1
- stoobly_agent/public/18-es2015.d3b430636a4d6f544d92.js +1 -0
- stoobly_agent/public/18-es5.d3b430636a4d6f544d92.js +1 -0
- stoobly_agent/public/35-es2015.f741ebce0bfc25f0ec99.js +1 -0
- stoobly_agent/public/35-es5.f741ebce0bfc25f0ec99.js +1 -0
- stoobly_agent/public/7-es2015.19ccb84e62e2ea874f53.js +1 -0
- stoobly_agent/public/7-es5.19ccb84e62e2ea874f53.js +1 -0
- stoobly_agent/public/9-es2015.b7bcad8238f58e214f03.js +1 -0
- stoobly_agent/public/9-es5.b7bcad8238f58e214f03.js +1 -0
- stoobly_agent/public/index.html +1 -1
- stoobly_agent/public/runtime-es2015.9addf49b79aca951b7e2.js +1 -0
- stoobly_agent/public/runtime-es5.9addf49b79aca951b7e2.js +1 -0
- stoobly_agent/test/app/cli/snapshot/snapshot_copy_test.py +56 -0
- stoobly_agent/test/app/cli/snapshot/snapshot_prune_test.py +2 -5
- stoobly_agent/test/app/cli/snapshot/snapshot_update_test.py +0 -1
- stoobly_agent/test/app/models/factories/resource/local_db/helpers/tiebreak_scenario_request_test.py +20 -2
- stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION +1 -1
- stoobly_agent/test/app/proxy/replay/body_parser_service_test.py +3 -3
- stoobly_agent/test/config/data_dir_test.py +4 -2
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/METADATA +2 -1
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/RECORD +73 -72
- stoobly_agent/public/18-es2015.10cdd5c608b10d90d19a.js +0 -1
- stoobly_agent/public/18-es5.10cdd5c608b10d90d19a.js +0 -1
- stoobly_agent/public/35-es2015.61a7ae8da93df94fab06.js +0 -1
- stoobly_agent/public/35-es5.61a7ae8da93df94fab06.js +0 -1
- stoobly_agent/public/7-es2015.c359dbb640e2af507221.js +0 -1
- stoobly_agent/public/7-es5.c359dbb640e2af507221.js +0 -1
- stoobly_agent/public/9-es2015.cfc1101139d6ae75731b.js +0 -1
- stoobly_agent/public/9-es5.cfc1101139d6ae75731b.js +0 -1
- stoobly_agent/public/runtime-es2015.08e65883d390cd16c15b.js +0 -1
- stoobly_agent/public/runtime-es5.08e65883d390cd16c15b.js +0 -1
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/LICENSE +0 -0
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/WHEEL +0 -0
- {stoobly_agent-0.34.9.dist-info → stoobly_agent-0.34.11.dist-info}/entry_points.txt +0 -0
stoobly_agent/__init__.py
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
COMMAND = 'stoobly-agent'
|
2
|
-
VERSION = '0.34.
|
2
|
+
VERSION = '0.34.11'
|
@@ -7,6 +7,8 @@ from stoobly_agent.config.constants import headers
|
|
7
7
|
from stoobly_agent.config.mitmproxy import MitmproxyConfig
|
8
8
|
from stoobly_agent.lib.logger import Logger
|
9
9
|
|
10
|
+
LOG_ID = 'ProxyController'
|
11
|
+
|
10
12
|
class ProxyController:
|
11
13
|
_instance = None
|
12
14
|
|
@@ -54,14 +56,14 @@ class ProxyController:
|
|
54
56
|
_params = context.params
|
55
57
|
_verify = not MitmproxyConfig.instance().get('ssl_insecure')
|
56
58
|
|
57
|
-
Logger.instance().debug('Request Headers')
|
58
|
-
Logger.instance().debug(_headers)
|
59
|
-
Logger.instance().debug('Cookies')
|
60
|
-
Logger.instance().debug(_cookies)
|
61
|
-
Logger.instance().debug('Body')
|
62
|
-
Logger.instance().debug(_body)
|
63
|
-
Logger.instance().debug('Query Params')
|
64
|
-
Logger.instance().debug(_params)
|
59
|
+
Logger.instance(LOG_ID).debug('Request Headers')
|
60
|
+
Logger.instance(LOG_ID).debug(_headers)
|
61
|
+
Logger.instance(LOG_ID).debug('Cookies')
|
62
|
+
Logger.instance(LOG_ID).debug(_cookies)
|
63
|
+
Logger.instance(LOG_ID).debug('Body')
|
64
|
+
Logger.instance(LOG_ID).debug(_body)
|
65
|
+
Logger.instance(LOG_ID).debug('Query Params')
|
66
|
+
Logger.instance(LOG_ID).debug(_params)
|
65
67
|
|
66
68
|
body = None
|
67
69
|
headers = {}
|
@@ -99,8 +101,8 @@ class ProxyController:
|
|
99
101
|
body = b'Unknown Error'
|
100
102
|
status = 0
|
101
103
|
|
102
|
-
Logger.instance().debug('Response Headers')
|
103
|
-
Logger.instance().debug(res.headers)
|
104
|
+
Logger.instance(LOG_ID).debug('Response Headers')
|
105
|
+
Logger.instance(LOG_ID).debug(res.headers)
|
104
106
|
|
105
107
|
context.render(
|
106
108
|
headers = headers,
|
@@ -23,6 +23,8 @@ from .helpers.handle_config_update_service import (
|
|
23
23
|
from .helpers.print_service import FORMATS, print_projects, print_scenarios, select_print_options
|
24
24
|
from .helpers.validations import *
|
25
25
|
|
26
|
+
LOG_ID = 'ConfigCLI'
|
27
|
+
|
26
28
|
settings = Settings.instance()
|
27
29
|
is_remote = settings.cli.features.remote or not not os.environ.get(env_vars.FEATURE_REMOTE)
|
28
30
|
|
@@ -261,7 +263,7 @@ def set(**kwargs):
|
|
261
263
|
|
262
264
|
settings.commit()
|
263
265
|
|
264
|
-
Logger.instance().debug(f"Rewrite {kwargs['name']} -> {kwargs['value']} set!")
|
266
|
+
Logger.instance(LOG_ID).debug(f"Rewrite {kwargs['name']} -> {kwargs['value']} set!")
|
265
267
|
|
266
268
|
### Match
|
267
269
|
|
@@ -326,7 +328,7 @@ def set(**kwargs):
|
|
326
328
|
|
327
329
|
settings.commit()
|
328
330
|
|
329
|
-
Logger.instance().debug(f"Match {kwargs['method']} {kwargs['pattern']} -> {kwargs['component']} set!")
|
331
|
+
Logger.instance(LOG_ID).debug(f"Match {kwargs['method']} {kwargs['pattern']} -> {kwargs['component']} set!")
|
330
332
|
|
331
333
|
### Firewall
|
332
334
|
|
@@ -392,7 +394,7 @@ def set(**kwargs):
|
|
392
394
|
|
393
395
|
settings.commit()
|
394
396
|
|
395
|
-
Logger.instance().debug(f"Firewall {kwargs['method']} {kwargs['pattern']} -> {kwargs['action']} set!")
|
397
|
+
Logger.instance(LOG_ID).debug(f"Firewall {kwargs['method']} {kwargs['pattern']} -> {kwargs['action']} set!")
|
396
398
|
|
397
399
|
### Validate
|
398
400
|
|
@@ -56,6 +56,11 @@ def handle_intercept_active_update(new_settings: Settings, context: Context = No
|
|
56
56
|
# If policy is overwrite when recording, whenever intercept is disabled,
|
57
57
|
# set active scenario to not be overwritable
|
58
58
|
scenario_model.update(_scenario_key.id, **{ 'overwritable': False })[1]
|
59
|
+
elif _mode == intercept_mode.MOCK:
|
60
|
+
# When mock is stopped, clear request access counts
|
61
|
+
from stoobly_agent.app.models.factories.resource.local_db.helpers.tiebreak_scenario_request import reset
|
62
|
+
|
63
|
+
reset()
|
59
64
|
|
60
65
|
def handle_scenario_update(new_settings: Settings, context = None):
|
61
66
|
new_scenario_key = __scenario_key(new_settings.proxy)
|
@@ -11,24 +11,26 @@ from typing import List
|
|
11
11
|
|
12
12
|
from stoobly_agent.app.models.adapters.raw_joined import RawJoinedRequestAdapterFactory
|
13
13
|
from stoobly_agent.app.models.factories.resource.local_db.helpers.log import Log
|
14
|
-
from stoobly_agent.app.models.factories.resource.local_db.helpers.log_event import LogEvent, REQUEST_RESOURCE, SCENARIO_RESOURCE
|
14
|
+
from stoobly_agent.app.models.factories.resource.local_db.helpers.log_event import LogEvent, PUT_ACTION, REQUEST_RESOURCE, SCENARIO_RESOURCE
|
15
15
|
from stoobly_agent.app.models.factories.resource.local_db.helpers.request_snapshot import RequestSnapshot
|
16
16
|
from stoobly_agent.app.models.factories.resource.local_db.helpers.scenario_snapshot import ScenarioSnapshot
|
17
17
|
from stoobly_agent.app.models.helpers.apply import Apply
|
18
|
+
from stoobly_agent.config.data_dir import DataDir
|
19
|
+
from stoobly_agent.lib.api.keys import RequestKey, ScenarioKey
|
18
20
|
|
19
21
|
from .helpers.print_service import FORMATS, print_snapshots, select_print_options
|
20
22
|
from .helpers.verify_raw_request_service import verify_raw_request
|
21
23
|
|
22
24
|
@click.group(
|
23
25
|
epilog="Run 'stoobly-agent project COMMAND --help' for more information on a command.",
|
24
|
-
help="Manage snapshots"
|
26
|
+
help="Manage snapshots."
|
25
27
|
)
|
26
28
|
@click.pass_context
|
27
29
|
def snapshot(ctx):
|
28
30
|
pass
|
29
31
|
|
30
32
|
@snapshot.command(
|
31
|
-
help="Apply snapshots",
|
33
|
+
help="Apply snapshots.",
|
32
34
|
)
|
33
35
|
@click.option('--force', default=False, help="Toggles whether resources are hard deleted.")
|
34
36
|
@click.argument('uuid', required=False)
|
@@ -41,7 +43,18 @@ def apply(**kwargs):
|
|
41
43
|
apply.all()
|
42
44
|
|
43
45
|
@snapshot.command(
|
44
|
-
help="
|
46
|
+
help="Copy snapshots to a different data directory."
|
47
|
+
)
|
48
|
+
@click.option('--request-key', multiple=True, help='')
|
49
|
+
@click.option('--scenario-key', multiple=True, help='')
|
50
|
+
@click.argument('destination', required=True)
|
51
|
+
def copy(**kwargs):
|
52
|
+
destination = kwargs['destination']
|
53
|
+
__copy_scenarios(kwargs['scenario_key'], destination)
|
54
|
+
__copy_requests(kwargs['request_key'], destination)
|
55
|
+
|
56
|
+
@snapshot.command(
|
57
|
+
help="List snapshots.",
|
45
58
|
name="list"
|
46
59
|
)
|
47
60
|
@click.option('--format', type=click.Choice(FORMATS), help='Format output.')
|
@@ -83,7 +96,7 @@ def prune(**kwargs):
|
|
83
96
|
log.prune(kwargs['dry_run'])
|
84
97
|
|
85
98
|
@snapshot.command(
|
86
|
-
help="Update snapshot",
|
99
|
+
help="Update snapshot.",
|
87
100
|
)
|
88
101
|
@click.option('--format', type=click.Choice(FORMATS), help='Format output.')
|
89
102
|
@click.option('--select', multiple=True, help='Select column(s) to display.')
|
@@ -266,4 +279,70 @@ def __transform_scenario(snapshot: ScenarioSnapshot):
|
|
266
279
|
event_dict['name'] = metadata.get('name')
|
267
280
|
event_dict['description'] = metadata.get('description')
|
268
281
|
|
269
|
-
return event_dict
|
282
|
+
return event_dict
|
283
|
+
|
284
|
+
def __copy_requests(request_keys: list, destination: str):
|
285
|
+
log = Log()
|
286
|
+
|
287
|
+
data_dir = DataDir.instance(destination)
|
288
|
+
destination_log = Log(data_dir)
|
289
|
+
|
290
|
+
for request_key in request_keys:
|
291
|
+
found = False
|
292
|
+
|
293
|
+
for event in log.target_events:
|
294
|
+
if event.action != PUT_ACTION:
|
295
|
+
continue
|
296
|
+
|
297
|
+
if event.resource != REQUEST_RESOURCE:
|
298
|
+
continue
|
299
|
+
|
300
|
+
key = RequestKey(request_key)
|
301
|
+
if event.resource_uuid != key.id:
|
302
|
+
continue
|
303
|
+
|
304
|
+
snapshot: RequestSnapshot = event.snapshot()
|
305
|
+
snapshot.copy(destination)
|
306
|
+
resource = snapshot.find_resource()
|
307
|
+
|
308
|
+
if not resource:
|
309
|
+
print(f"Could not find request {key.id}", file=sys.stderr)
|
310
|
+
else:
|
311
|
+
destination_log.put(resource)
|
312
|
+
found = True
|
313
|
+
|
314
|
+
if not found:
|
315
|
+
print(f"No snapshot found for {key}", file=sys.stderr)
|
316
|
+
|
317
|
+
def __copy_scenarios(scenario_keys: list, destination: str):
|
318
|
+
log = Log()
|
319
|
+
|
320
|
+
data_dir = DataDir.instance(destination)
|
321
|
+
destination_log = Log(data_dir)
|
322
|
+
|
323
|
+
for scenario_key in scenario_keys:
|
324
|
+
found = False
|
325
|
+
|
326
|
+
for event in log.target_events:
|
327
|
+
if event.action != PUT_ACTION:
|
328
|
+
continue
|
329
|
+
|
330
|
+
if event.resource != SCENARIO_RESOURCE:
|
331
|
+
continue
|
332
|
+
|
333
|
+
key = ScenarioKey(scenario_key)
|
334
|
+
if event.resource_uuid != key.id:
|
335
|
+
continue
|
336
|
+
|
337
|
+
snapshot: ScenarioSnapshot = event.snapshot()
|
338
|
+
snapshot.copy(destination)
|
339
|
+
resource = snapshot.find_resource()
|
340
|
+
|
341
|
+
if not resource:
|
342
|
+
print(f"Could not find scenario {key.id}", file=sys.stderr)
|
343
|
+
else:
|
344
|
+
destination_log.put(resource)
|
345
|
+
found = True
|
346
|
+
|
347
|
+
if not found:
|
348
|
+
print(f"No snapshot found for {key}", file=sys.stderr)
|
@@ -11,11 +11,13 @@ from .log_event import LogEvent
|
|
11
11
|
from .snapshot_types import DELETE_ACTION, PUT_ACTION, Resource
|
12
12
|
|
13
13
|
EVENT_DELIMITTER = "\n"
|
14
|
+
LOG_ID = 'Log'
|
14
15
|
|
15
16
|
class Log():
|
16
17
|
|
17
|
-
def __init__(self):
|
18
|
-
data_dir = DataDir.instance()
|
18
|
+
def __init__(self, data_dir: DataDir = None):
|
19
|
+
data_dir = data_dir or DataDir.instance()
|
20
|
+
|
19
21
|
self.__log_file_path = data_dir.snapshots_log_file_path
|
20
22
|
self.__history_dir_path = data_dir.snapshots_history_dir_path
|
21
23
|
|
@@ -194,7 +196,7 @@ class Log():
|
|
194
196
|
snapshot_exists = snapshot.exists
|
195
197
|
|
196
198
|
if event.action == DELETE_ACTION or not snapshot_exists:
|
197
|
-
Logger.instance().info(f"{bcolors.OKBLUE}Removing {event.resource} {event.resource_uuid}
|
199
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Removing{bcolors.ENDC} {event.resource} {event.resource_uuid}")
|
198
200
|
|
199
201
|
resource_events: List[LogEvent] = resource_index[event.resource_uuid]
|
200
202
|
removed_events = {}
|
@@ -204,14 +206,14 @@ class Log():
|
|
204
206
|
if event.uuid in removed_events:
|
205
207
|
continue
|
206
208
|
|
207
|
-
Logger.instance().info(f"Removing event {event.uuid}")
|
209
|
+
Logger.instance(LOG_ID).info(f"Removing event {event.uuid}")
|
208
210
|
self.remove_event_history(event, history_path, dry_run)
|
209
211
|
removed_events[event.uuid] = True
|
210
212
|
|
211
213
|
if event.action == DELETE_ACTION and snapshot_exists:
|
212
214
|
if not dry_run:
|
213
215
|
snapshot.remove()
|
214
|
-
Logger.instance().info(f"Removing {event.resource} snapshot")
|
216
|
+
Logger.instance(LOG_ID).info(f"Removing {event.resource} snapshot")
|
215
217
|
|
216
218
|
def build_raw_events(self, contents: str) -> List[str]:
|
217
219
|
if not contents:
|
@@ -259,13 +261,13 @@ class Log():
|
|
259
261
|
events = list(filter(lambda log_event: log_event.uuid != event.uuid, events))
|
260
262
|
|
261
263
|
if len(events) == 0:
|
262
|
-
Logger.instance().info(f"Removing {history_path}")
|
264
|
+
Logger.instance(LOG_ID).info(f"Removing {history_path}")
|
263
265
|
|
264
266
|
if not dry_run:
|
265
267
|
os.remove(history_path)
|
266
268
|
else:
|
267
269
|
new_raw_events = list(map(lambda event: str(event), events))
|
268
|
-
Logger.instance().info(f"Updating {history_path}, Events: {len(raw_events)} -> {len(new_raw_events)}")
|
270
|
+
Logger.instance(LOG_ID).info(f"Updating {history_path}, Events: {len(raw_events)} -> {len(new_raw_events)}")
|
269
271
|
|
270
272
|
if not dry_run:
|
271
273
|
with open(history_path, 'w') as fp:
|
@@ -1,4 +1,5 @@
|
|
1
1
|
import os
|
2
|
+
import shutil
|
2
3
|
|
3
4
|
from stoobly_agent.app.models.adapters.orm import JoinedRequestStringAdapter
|
4
5
|
from stoobly_agent.lib.orm.request import Request
|
@@ -47,6 +48,10 @@ class RequestSnapshot(Snapshot):
|
|
47
48
|
with open(self.path, 'rb') as fp:
|
48
49
|
self.__backup = fp.read()
|
49
50
|
|
51
|
+
def copy(self, dest_dir: str):
|
52
|
+
request_file_path = self.path
|
53
|
+
return self.copy_file(request_file_path, dest_dir)
|
54
|
+
|
50
55
|
def find_resource(self):
|
51
56
|
return Request.find_by(uuid=self.uuid)
|
52
57
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import json
|
2
2
|
import os
|
3
3
|
import pdb
|
4
|
+
import shutil
|
4
5
|
|
5
6
|
from typing import Callable
|
6
7
|
|
@@ -74,6 +75,26 @@ class ScenarioSnapshot(Snapshot):
|
|
74
75
|
|
75
76
|
self.iter_request_snapshots(self.__handle_backup_requests)
|
76
77
|
|
78
|
+
def copy(self, destination):
|
79
|
+
self.copy_metadata(destination)
|
80
|
+
self.copy_requests(destination)
|
81
|
+
|
82
|
+
def copy_metadata(self, dest_dir: str):
|
83
|
+
metadata_file_path = self.metadata_path
|
84
|
+
return self.copy_file(metadata_file_path, dest_dir)
|
85
|
+
|
86
|
+
def copy_requests(self, dest_dir: str):
|
87
|
+
if not os.path.exists(dest_dir):
|
88
|
+
os.makedirs(dest_dir, exist_ok=True)
|
89
|
+
|
90
|
+
requests_file_path = self.requests_path
|
91
|
+
|
92
|
+
if os.path.exists(requests_file_path):
|
93
|
+
# A request only ever belongs to one scenario
|
94
|
+
self.iter_request_snapshots(lambda snapshot: self.__handle_copy_requests(snapshot, dest_dir))
|
95
|
+
|
96
|
+
self.copy_file(requests_file_path, dest_dir)
|
97
|
+
|
77
98
|
def iter_request_snapshots(self, handler: Callable[[RequestSnapshot], None]):
|
78
99
|
requests_file_path = self.requests_path
|
79
100
|
|
@@ -136,5 +157,8 @@ class ScenarioSnapshot(Snapshot):
|
|
136
157
|
def __handle_backup_requests(self, request_snapshot: RequestSnapshot):
|
137
158
|
self.__requests_backup[request_snapshot.uuid] = request_snapshot.request
|
138
159
|
|
160
|
+
def __handle_copy_requests(self, request_snapshot: RequestSnapshot, dest_dir: str):
|
161
|
+
request_snapshot.copy(dest_dir)
|
162
|
+
|
139
163
|
def __handle_remove_requests(self, request_snapshot: RequestSnapshot):
|
140
164
|
request_snapshot.remove()
|
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
import os
|
2
|
+
import shutil
|
2
3
|
|
3
4
|
from stoobly_agent.config.data_dir import DataDir
|
4
5
|
|
@@ -14,4 +15,19 @@ class Snapshot():
|
|
14
15
|
|
15
16
|
@property
|
16
17
|
def data_dir(self):
|
17
|
-
return self.__data_dir
|
18
|
+
return self.__data_dir
|
19
|
+
|
20
|
+
def copy_file(self, src: str, dest_dir: str):
|
21
|
+
if not os.path.exists(src):
|
22
|
+
return None
|
23
|
+
|
24
|
+
data_dir_parent = os.path.dirname(self.data_dir.path)
|
25
|
+
dest_file_path = src.replace(data_dir_parent, dest_dir)
|
26
|
+
dest_dir_path = os.path.dirname(dest_file_path)
|
27
|
+
|
28
|
+
if not os.path.exists(dest_dir_path):
|
29
|
+
os.makedirs(dest_dir_path, exist_ok=True)
|
30
|
+
|
31
|
+
shutil.copy(src, dest_file_path)
|
32
|
+
|
33
|
+
return dest_file_path
|
@@ -5,6 +5,8 @@ from typing import List
|
|
5
5
|
from stoobly_agent.lib.cache import Cache
|
6
6
|
from stoobly_agent.lib.orm.request import Request
|
7
7
|
|
8
|
+
PREFIX = 'last_request_id'
|
9
|
+
|
8
10
|
def access_request(session_id: str, request_id: int, timeout = None):
|
9
11
|
cache = Cache.instance()
|
10
12
|
_last_request_id_key = __last_request_id_key(session_id)
|
@@ -18,6 +20,9 @@ def generate_session_id(query: dict):
|
|
18
20
|
|
19
21
|
return hashlib.md5(b'.'.join(toks)).hexdigest()
|
20
22
|
|
23
|
+
def reset():
|
24
|
+
Cache.instance().clear(f".+\.{PREFIX}")
|
25
|
+
|
21
26
|
def tiebreak_scenario_request(session_id: str, requests: List[Request]):
|
22
27
|
if len(requests) == 0:
|
23
28
|
return None
|
@@ -43,4 +48,4 @@ def tiebreak_scenario_request(session_id: str, requests: List[Request]):
|
|
43
48
|
|
44
49
|
def __last_request_id_key(_key = None):
|
45
50
|
_key = _key or generate_session_id()
|
46
|
-
return f"{_key}.
|
51
|
+
return f"{_key}.{PREFIX}"
|
@@ -58,12 +58,16 @@ class LocalDBQueryParamAdapter(LocalDBAdapter):
|
|
58
58
|
if not name in _query_params:
|
59
59
|
_query_params[name] = []
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
61
|
+
if isinstance(_query_params[name], list):
|
62
|
+
try:
|
63
|
+
index = _query_params[name].index(decoded_id['value'])
|
64
|
+
except ValueError as e:
|
65
|
+
return self.not_found()
|
66
|
+
|
67
|
+
_query_params[name][index] = value
|
68
|
+
else:
|
69
|
+
_query_params[name] = value
|
65
70
|
|
66
|
-
_query_params[name][index] = value
|
67
71
|
parsed_url = parsed_url._replace(query=urlencode(_query_params, True))
|
68
72
|
|
69
73
|
request = LocalDBRequestAdapter(self.__request_orm).update(request_id, url=parsed_url.geturl())
|
@@ -112,6 +112,7 @@ class LocalDBRequestAdapter(LocalDBAdapter):
|
|
112
112
|
|
113
113
|
headers = {}
|
114
114
|
headers[custom_headers.MOCK_REQUEST_ID] = str(request.id)
|
115
|
+
headers[custom_headers.MOCK_REQUEST_KEY] = request.key()
|
115
116
|
headers[custom_headers.RESPONSE_LATENCY] = str(request.latency)
|
116
117
|
|
117
118
|
return (
|
@@ -10,7 +10,7 @@ from stoobly_agent.app.models.request_model import RequestModel
|
|
10
10
|
from stoobly_agent.app.proxy.mitmproxy.request_facade import MitmproxyRequestFacade
|
11
11
|
from stoobly_agent.app.proxy.utils.rewrite_rules_to_ignored_components_service import rewrite_rules_to_ignored_components
|
12
12
|
from stoobly_agent.config.constants import custom_headers, env_vars, lifecycle_hooks, mock_policy, request_origin
|
13
|
-
from stoobly_agent.lib.logger import Logger
|
13
|
+
from stoobly_agent.lib.logger import bcolors, Logger
|
14
14
|
|
15
15
|
from .constants import custom_response_codes
|
16
16
|
from .mock.context import MockContext
|
@@ -20,7 +20,7 @@ from .utils.allowed_request_service import get_active_mode_policy
|
|
20
20
|
from .utils.request_handler import reverse_proxy
|
21
21
|
from .utils.response_handler import bad_request, pass_on
|
22
22
|
|
23
|
-
LOG_ID = '
|
23
|
+
LOG_ID = 'Mock'
|
24
24
|
|
25
25
|
class MockOptions(TypedDict):
|
26
26
|
failure: Callable
|
@@ -124,11 +124,18 @@ def handle_request_mock(context: MockContext):
|
|
124
124
|
success=lambda context: __handle_mock_success(context)
|
125
125
|
)
|
126
126
|
|
127
|
-
def
|
128
|
-
response = context.response
|
129
|
-
|
127
|
+
def handle_response_mock(context: MockContext):
|
128
|
+
response = context.flow.response
|
129
|
+
request_key = response.headers.get(custom_headers.MOCK_REQUEST_KEY)
|
130
|
+
|
131
|
+
if request_key:
|
132
|
+
request = context.flow.request
|
133
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Mocked{bcolors.ENDC} {request.url} -> {request_key}")
|
130
134
|
|
135
|
+
def __handle_mock_success(context: MockContext) -> None:
|
131
136
|
if os.environ.get(env_vars.AGENT_SIMULATE_LATENCY):
|
137
|
+
response = context.response
|
138
|
+
start_time = context.start_time
|
132
139
|
__simulate_latency(response.headers.get(custom_headers.RESPONSE_LATENCY), start_time)
|
133
140
|
|
134
141
|
def __handle_mock_failure(context: MockContext) -> None:
|
@@ -143,7 +150,7 @@ def __handle_mock_failure(context: MockContext) -> None:
|
|
143
150
|
else:
|
144
151
|
req.headers[custom_headers.REQUEST_ORIGIN] = request_origin.PROXY
|
145
152
|
|
146
|
-
Logger.instance().debug(f"
|
153
|
+
Logger.instance(LOG_ID).debug(f"UpstreamUrl: {upstream_url}")
|
147
154
|
|
148
155
|
reverse_proxy(req, upstream_url, {})
|
149
156
|
|
@@ -167,9 +174,9 @@ def __simulate_latency(expected_latency: str, start_time: float) -> float:
|
|
167
174
|
|
168
175
|
wait_time = expected_latency - estimated_rtt_network_latency - api_latency
|
169
176
|
|
170
|
-
Logger.instance().debug(f"
|
171
|
-
Logger.instance().debug(f"
|
172
|
-
Logger.instance().debug(f"
|
177
|
+
Logger.instance(LOG_ID).debug(f"Expected latency: {expected_latency}")
|
178
|
+
Logger.instance(LOG_ID).debug(f"API latency: {api_latency}")
|
179
|
+
Logger.instance(LOG_ID).debug(f"Wait time: {wait_time}")
|
173
180
|
|
174
181
|
if wait_time > 0:
|
175
182
|
time.sleep(wait_time)
|
@@ -19,7 +19,7 @@ from .record.upload_request_service import inject_upload_request
|
|
19
19
|
from .utils.allowed_request_service import get_active_mode_policy
|
20
20
|
from .utils.response_handler import bad_request, disable_transfer_encoding
|
21
21
|
|
22
|
-
LOG_ID = '
|
22
|
+
LOG_ID = 'Record'
|
23
23
|
|
24
24
|
def handle_response_record(context: RecordContext):
|
25
25
|
flow = context.flow
|
@@ -32,7 +32,7 @@ def handle_response_record(context: RecordContext):
|
|
32
32
|
request_model = RequestModel(intercept_settings.settings)
|
33
33
|
|
34
34
|
active_record_policy = get_active_mode_policy(request, intercept_settings)
|
35
|
-
Logger.instance().debug(f"
|
35
|
+
Logger.instance(LOG_ID).debug(f"RecordPolicy: {active_record_policy}")
|
36
36
|
|
37
37
|
if active_record_policy == record_policy.ALL:
|
38
38
|
__record_request(context, request_model)
|
@@ -5,7 +5,7 @@ from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
|
|
5
5
|
from mitmproxy.http import Headers, Request as MitmproxyRequest
|
6
6
|
|
7
7
|
from stoobly_agent.app.proxy.context import InterceptContext
|
8
|
-
from stoobly_agent.app.proxy.handle_mock_service import handle_request_mock
|
8
|
+
from stoobly_agent.app.proxy.handle_mock_service import handle_request_mock, handle_response_mock
|
9
9
|
from stoobly_agent.app.proxy.handle_replay_service import handle_request_replay, handle_response_replay
|
10
10
|
from stoobly_agent.app.proxy.handle_record_service import handle_response_record
|
11
11
|
from stoobly_agent.app.proxy.handle_test_service import handle_request_test, handle_response_test
|
@@ -21,7 +21,7 @@ from stoobly_agent.lib.logger import Logger
|
|
21
21
|
# Disable proxy settings in urllib
|
22
22
|
os.environ['no_proxy'] = '*'
|
23
23
|
|
24
|
-
LOG_ID = '
|
24
|
+
LOG_ID = 'Intercept'
|
25
25
|
|
26
26
|
def request(flow: MitmproxyHTTPFlow):
|
27
27
|
request: MitmproxyRequest = flow.request
|
@@ -36,7 +36,7 @@ def request(flow: MitmproxyHTTPFlow):
|
|
36
36
|
__intercept_hook(lifecycle_hooks.BEFORE_REQUEST, flow, intercept_settings)
|
37
37
|
|
38
38
|
active_mode = intercept_settings.mode
|
39
|
-
Logger.instance().debug(f"
|
39
|
+
Logger.instance(LOG_ID).debug(f"ProxyMode: {active_mode}")
|
40
40
|
|
41
41
|
if active_mode == mode.MOCK:
|
42
42
|
context = MockContext(flow, intercept_settings)
|
@@ -69,7 +69,10 @@ def response(flow: MitmproxyHTTPFlow):
|
|
69
69
|
|
70
70
|
active_mode = intercept_settings.mode
|
71
71
|
|
72
|
-
if active_mode == mode.
|
72
|
+
if active_mode == mode.MOCK:
|
73
|
+
context = MockContext(flow, intercept_settings)
|
74
|
+
return handle_response_mock(context)
|
75
|
+
elif active_mode == mode.RECORD:
|
73
76
|
context = RecordContext(flow, intercept_settings)
|
74
77
|
return handle_response_record(context)
|
75
78
|
elif active_mode == mode.REPLAY:
|
@@ -17,6 +17,8 @@ from stoobly_agent.lib.utils.decode import decode
|
|
17
17
|
from .request_body_facade import MitmproxyRequestBodyFacade
|
18
18
|
from .request import Request
|
19
19
|
|
20
|
+
LOG_ID = 'Request'
|
21
|
+
|
20
22
|
class MitmproxyRequestFacade(Request):
|
21
23
|
|
22
24
|
###
|
@@ -137,7 +139,7 @@ class MitmproxyRequestFacade(Request):
|
|
137
139
|
|
138
140
|
if len(rewrites):
|
139
141
|
self.__rewrite_url(rewrites)
|
140
|
-
Logger.instance().debug(f"{bcolors.OKBLUE} Rewritten URL{bcolors.ENDC} {self.url}")
|
142
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE} Rewritten URL{bcolors.ENDC} {self.url}")
|
141
143
|
|
142
144
|
# Find all the rules that match request url and method
|
143
145
|
def select_rewrite_rules(self, rules: List[RewriteRule]) -> List[RewriteRule]:
|
@@ -186,7 +188,7 @@ class MitmproxyRequestFacade(Request):
|
|
186
188
|
})
|
187
189
|
|
188
190
|
def __rewrite_handler(self, rewrite: ParameterRule) -> str:
|
189
|
-
Logger.instance().info(f"{bcolors.OKCYAN}Rewriting {rewrite.type.lower()}
|
191
|
+
Logger.instance(LOG_ID).info(f"{bcolors.OKCYAN}Rewriting{bcolors.ENDC} {rewrite.type.lower()} {rewrite.name} => {rewrite.value}")
|
190
192
|
return rewrite.value
|
191
193
|
|
192
194
|
def __rewrite_url(self, rewrites: List[UrlRule]):
|
@@ -11,6 +11,8 @@ from stoobly_agent.lib.logger import bcolors, Logger
|
|
11
11
|
|
12
12
|
from .types import Fixtures
|
13
13
|
|
14
|
+
LOG_ID = 'Fixture'
|
15
|
+
|
14
16
|
class Options():
|
15
17
|
public_directory_path: str
|
16
18
|
response_fixtures: Fixtures
|
@@ -44,7 +46,7 @@ def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Respon
|
|
44
46
|
response.raw = BytesIO(fp.read())
|
45
47
|
response.headers = headers
|
46
48
|
|
47
|
-
Logger.instance().debug(f"{bcolors.OKBLUE}Resolved fixture {fixture_path}
|
49
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE}Resolved{bcolors.ENDC} fixture {fixture_path}")
|
48
50
|
|
49
51
|
return response
|
50
52
|
|
@@ -17,9 +17,9 @@ COMPONENT_TYPES = {
|
|
17
17
|
'RESPONSE': 5
|
18
18
|
}
|
19
19
|
|
20
|
-
|
20
|
+
LOG_ID = 'HashedRequest'
|
21
21
|
|
22
|
-
|
22
|
+
class HashedRequestDecorator:
|
23
23
|
|
24
24
|
def __init__(self, request: MitmproxyRequestFacade):
|
25
25
|
self.request = request
|
@@ -53,8 +53,8 @@ class HashedRequestDecorator:
|
|
53
53
|
|
54
54
|
ignored_headers = {} if with_ignored else self.ignored_headers
|
55
55
|
|
56
|
-
Logger.instance().debug(f"{bcolors.OKCYAN}Hashing headers...{bcolors.ENDC}")
|
57
|
-
Logger.instance().debug(f"{bcolors.OKBLUE}Ignoring{bcolors.ENDC} {ignored_headers}")
|
56
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKCYAN}Hashing headers...{bcolors.ENDC}")
|
57
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE}Ignoring{bcolors.ENDC} {ignored_headers}")
|
58
58
|
serialized_params = self.__serialize_params(headers, ignored_headers)
|
59
59
|
|
60
60
|
return self.__hash_serialized_params(serialized_params)
|
@@ -68,8 +68,8 @@ class HashedRequestDecorator:
|
|
68
68
|
params = self.__deflatten_multi_dict(query_params)
|
69
69
|
ignored_params = {} if with_ignored else self.ignored_query_params
|
70
70
|
|
71
|
-
Logger.instance().debug(f"{bcolors.OKCYAN}Hashing query params...{bcolors.ENDC}")
|
72
|
-
Logger.instance().debug(f"{bcolors.OKBLUE}Ignoring{bcolors.ENDC} {ignored_params}")
|
71
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKCYAN}Hashing query params...{bcolors.ENDC}")
|
72
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE}Ignoring{bcolors.ENDC} {ignored_params}")
|
73
73
|
serialized_params = self.__serialize_params(params, ignored_params)
|
74
74
|
|
75
75
|
return self.__hash_serialized_params(serialized_params)
|
@@ -81,8 +81,8 @@ class HashedRequestDecorator:
|
|
81
81
|
params = self.request.parsed_body
|
82
82
|
ignored_params = {} if with_ignored else self.ignored_body_params
|
83
83
|
|
84
|
-
Logger.instance().debug(f"{bcolors.OKCYAN}Hashing body params...{bcolors.ENDC}")
|
85
|
-
Logger.instance().debug(f"{bcolors.OKBLUE}Ignoring{bcolors.ENDC} {ignored_params}")
|
84
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKCYAN}Hashing body params...{bcolors.ENDC}")
|
85
|
+
Logger.instance(LOG_ID).debug(f"{bcolors.OKBLUE}Ignoring{bcolors.ENDC} {ignored_params}")
|
86
86
|
|
87
87
|
return RequestHasher.instance().hash_params(params, ignored_params)
|
88
88
|
|
@@ -120,13 +120,13 @@ class HashedRequestDecorator:
|
|
120
120
|
for param in value:
|
121
121
|
param_hash = hashlib.md5(self.__serialize_param(key, param)).hexdigest()
|
122
122
|
|
123
|
-
Logger.instance().debug(f"{
|
123
|
+
Logger.instance(LOG_ID).debug(f"Serializing {key} -> {param} ({param_hash})")
|
124
124
|
|
125
125
|
serialized_params.append(param_hash)
|
126
126
|
else:
|
127
127
|
param_hash = hashlib.md5(self.__serialize_param(key, value)).hexdigest()
|
128
128
|
|
129
|
-
Logger.instance().debug(f"{
|
129
|
+
Logger.instance(LOG_ID).debug(f"Serializing {key} -> {value} ({param_hash})")
|
130
130
|
|
131
131
|
serialized_params.append(param_hash)
|
132
132
|
|