stoobly-agent 1.9.0__tar.gz → 1.9.2__tar.gz

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: stoobly-agent
-Version: 1.9.0
+Version: 1.9.2
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/pyproject.toml

@@ -10,7 +10,7 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "1.9.0"
+version = "1.9.2"
 
 [tool.poetry.dependencies]
 python = "^3.10"

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '1.9.0'
+VERSION = '1.9.2'

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/app/models/factories/resource/local_db/helpers/log.py

@@ -8,6 +8,7 @@ from stoobly_agent.config.data_dir import DataDir
 from stoobly_agent.lib.logger import bcolors, Logger
 
 from .log_event import LogEvent
+from .request_snapshot import RequestSnapshot
 from .snapshot_types import DELETE_ACTION, PUT_ACTION, Resource
 
 EVENT_DELIMITTER = "\n"
@@ -67,6 +68,10 @@ class Log():
     contents = self.read()
     return self.build_raw_events(contents) 
 
+  @property
+  def scenario_inverted_index(self):
+    return self.build_scenario_inverted_index(self.target_events)
+
   @property
   def unprocessed_events(self) -> List[LogEvent]:
     events = self.events
@@ -133,6 +138,22 @@ class Log():
 
   def build_log_events(self, raw_events) -> List[LogEvent]:
     return list(map(lambda raw_event: LogEvent(raw_event), raw_events))
+  
+  def build_scenario_inverted_index(self, events: List[LogEvent], index = {}):
+    def handle_snapshot(snapshot: RequestSnapshot):
+      request_uuid = snapshot.uuid
+      if not request_uuid in index:
+        index[request_uuid] = []
+
+      index[request_uuid].append(event.resource_uuid)
+
+    for event in events:
+      if not event.is_scenario():
+        continue 
+
+      event.snapshot().iter_request_snapshots(handle_snapshot)
+
+    return index
 
   def next_version(self, last_processed_uuid: str = None):
     uuids = self.uuids()
@@ -149,7 +170,7 @@ class Log():
   def collapse(self, events: List[LogEvent]) -> List[LogEvent]:
     events_count = {}
 
-    # More recent events take precedence over earlier ones, keep only the most recent event 
+    # More recent events take precedence over earlier ones, only the most recent event 
     for event in events:
       event_key = event.key
 
@@ -197,12 +218,18 @@ class Log():
 
       resource_index[event.resource_uuid].append(event)
 
+    scenario_inverted_index = self.scenario_inverted_index
+
     pruned_events = self.collapse(events)
     for event in pruned_events:
       snapshot = event.snapshot()
       snapshot_exists = snapshot.exists
 
       if event.action == DELETE_ACTION or not snapshot_exists:
+        if event.is_request() and event.resource_uuid in scenario_inverted_index:
+          # If a request is deleted, only prune if it's not also a part of a scenario
+          continue
+
         Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Removing{bcolors.ENDC} {event.resource} {event.resource_uuid}")
 
         resource_events: List[LogEvent] = resource_index[event.resource_uuid]
@@ -218,8 +245,16 @@ class Log():
           removed_events[event.uuid] = True
 
         if event.action == DELETE_ACTION and snapshot_exists: 
-          if not dry_run:
+          if dry_run:
+            continue
+
+          if event.is_scenario():
+            # We still need to check each request in a scenario to make sure another scenario does not depend on it
+            snapshot.remove(lambda snapshot: self.remove_request_snapshot(snapshot, scenario_inverted_index))
+          elif event.is_request():
+            # We have already checked that a scenario does not depend on the request above
             snapshot.remove()
+
           Logger.instance(LOG_ID).info(f"Removing {event.resource} snapshot")
 
   def build_raw_events(self, contents: str) -> List[str]:
@@ -240,8 +275,10 @@ class Log():
   
   def remove_dangling_events(self, processed_events: List[LogEvent], unprocessed_events: List[LogEvent]):
     '''
-    Remove DELETE events where the last processed event was a PUT
+    Remove DELETE events unless the last processed event was a PUT
     '''
+
+    # Build an index to keep track of the last action that occurred for a resource
     index = {}
     for event in processed_events:
       if event.action == PUT_ACTION:
@@ -249,10 +286,32 @@ class Log():
       elif event.action == DELETE_ACTION:
         if event.resource_uuid in index:
           del index[event.resource_uuid]
+
+    scenario_inverted_index = self.build_scenario_inverted_index(processed_events)
+    
+    def keep(e: LogEvent):
+      # Keep the event if it's a PUT, it may have been updated
+      if e.action == PUT_ACTION:
+        return True
+
+      if e.action == DELETE_ACTION:
+        if e.is_request():
+          referenced = e.resource_uuid in scenario_inverted_index
+          # Keep the DELETE event if the requests exists in a scenario
+          # or if it doesn't exist in a scenario, there was a previous delete event
+          return referenced or (e.resource_uuid in index and not referenced)
+        elif e.is_scenario():
+          # Update scenario_inverted_index with unprocessed event
+          self.build_scenario_inverted_index([e], scenario_inverted_index)
+
+          # Keep DELETE scenario event only if previous event for the resource was PUT
+          return e.resource_uuid in index
+        else:
+          return True
     
     return list(
       filter(
-        lambda e: e.action != DELETE_ACTION or (e.action == DELETE_ACTION and e.resource_uuid in index), 
+        keep,
         unprocessed_events
       )
     )
@@ -289,6 +348,12 @@ class Log():
     remaining_events = self.remove_dangling_events(processed_events, unprocessed_events)
     return list(filter(lambda e: e.uuid not in remaining_version_uuids, remaining_events))
 
+  def remove_request_snapshot(self, snapshot: RequestSnapshot, scenario_inverted_index: dict = None):
+    scenario_inverted_index = scenario_inverted_index or self.scenario_inverted_index
+
+    if snapshot.uuid not in scenario_inverted_index:
+      snapshot.remove()
+
   # Rotate log to history
   def rotate(self):
     if not os.path.exists(self.__log_file_path):

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/app/models/factories/resource/local_db/helpers/request_snapshot.py

@@ -1,5 +1,4 @@
 import os
-import shutil
 
 from stoobly_agent.app.models.adapters.orm import JoinedRequestStringAdapter
 from stoobly_agent.lib.orm.request import Request

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/app/models/factories/resource/local_db/helpers/scenario_snapshot.py

@@ -1,7 +1,6 @@
 import json
 import os
 import pdb
-import shutil
 
 from typing import Callable, List
 
@@ -128,12 +127,12 @@ class ScenarioSnapshot(Snapshot):
     if os.path.exists(metadata_file_path):
       os.remove(metadata_file_path)
 
-  def remove_requests(self):
+  def remove_requests(self, handle_remove_request_snapshot = None):
     requests_file_path = self.requests_path
 
     if os.path.exists(requests_file_path):
       # A request only every belongs to one scenario
-      self.iter_request_snapshots(self.__handle_remove_requests)
+      self.iter_request_snapshots(handle_remove_request_snapshot or self.__handle_remove_requests)
 
       os.remove(requests_file_path)
 

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/app/models/factories/resource/local_db/helpers/snapshot_service.py

@@ -14,15 +14,17 @@ def snapshot_request(request: Request, **options: RequestSnapshotOptions):
     return
 
   snapshot = RequestSnapshot(request.uuid)
-
   snapshot.backup()
 
+  log = Log()
+
   if action == PUT_ACTION:
     snapshot.write(request, **options)
   elif action == DELETE_ACTION:
-    snapshot.remove()
+    inverted_scenario_index = log.scenario_inverted_index
 
-  log = Log()
+    # If a scenario currently depends on this request, we can't remove the snapshot until the scenario is also removed
+    log.remove_request_snapshot(snapshot, inverted_scenario_index) 
 
   try:
     if action == PUT_ACTION:
@@ -40,7 +42,6 @@ def snapshot_scenario(scenario: Scenario, **options):
     return
 
   snapshot = ScenarioSnapshot(scenario.uuid)
-
   snapshot.backup_metadata()
 
   if action == PUT_ACTION:
@@ -50,13 +51,13 @@ def snapshot_scenario(scenario: Scenario, **options):
 
   snapshot.backup_requests()
 
+  log = Log()
+  inverted_scenario_index = log.scenario_inverted_index
   if action == PUT_ACTION:
-    snapshot.remove_requests()
+    snapshot.remove_requests(lambda snapshot: log.remove_request_snapshot(snapshot, inverted_scenario_index))
     snapshot.write_requests(scenario, **options)
   elif action == DELETE_ACTION:
-    snapshot.remove_requests()
-
-  log = Log()
+    snapshot.remove_requests(lambda snapshot: log.remove_request_snapshot(snapshot, inverted_scenario_index))
 
   if action == PUT_ACTION:
     log.put(scenario)

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/app/models/helpers/apply.py

@@ -1,5 +1,7 @@
 import pdb
 
+from typing import List
+
 from stoobly_agent.app.models.factories.resource.local_db.helpers.log import Log
 from stoobly_agent.app.models.factories.resource.local_db.helpers.request_snapshot import RequestSnapshot
 from stoobly_agent.app.models.factories.resource.local_db.helpers.scenario_snapshot import ScenarioSnapshot
@@ -71,6 +73,7 @@ class Apply():
       if results:
         status = results[1]
         if status == 0 or status >= 400:
+          self.__logger(f"{bcolors.FAIL}Error{bcolors.ENDC} {results[0]}")
           completed = False
           break
 
@@ -218,14 +221,16 @@ class Apply():
 
     snapshot_requests = {}
 
-    request_snapshots = snapshot.request_snapshots
+    request_snapshots: List[RequestSnapshot] = snapshot.request_snapshots
     for request_snapshot in request_snapshots:
       raw_request = request_snapshot.request
-      
-      toks = raw_request.split(REQUEST_STRING_CLRF, 1)
 
+      if not raw_request:
+        return f"{request_snapshot.path} is missing", 400
+        
+      toks = raw_request.split(REQUEST_STRING_CLRF, 1)
       if len(toks) != 2:
-        return f"{snapshot.requests_path} contains an invalid request", 400
+        return f"{request_snapshot.path} contains an invalid request", 400
 
       uuid = request_snapshot.uuid
       res, status = self.__put_request(uuid, raw_request, scenario_id=scenario['id'])

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/app/proxy/handle_mock_service.py

@@ -147,15 +147,8 @@ def handle_request_mock(context: MockContext):
 # 2. AFTER_MOCK gets triggered (if mock found)
 #
 def handle_response_mock(context: MockContext):
-    response = context.flow.response
-    request_key = response.headers.get(custom_headers.MOCK_REQUEST_KEY)
-
-    if request_key:
-        request = context.flow.request
-        Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Mocked{bcolors.ENDC} {request.url} -> {request_key}")
-
-        __rewrite_response(context)
-        __mock_hook(lifecycle_hooks.AFTER_MOCK, context)
+    __rewrite_response(context)
+    __mock_hook(lifecycle_hooks.AFTER_MOCK, context)
 
 def __handle_mock_failure(context: MockContext) -> None:
     flow = context.flow
@@ -186,6 +179,19 @@ def __handle_found_policy(context: MockContext) -> None:
     reverse_proxy(req, upstream_url, {})
 
 def __handle_mock_success(context: MockContext) -> None:
+    response = context.response
+
+    if response:
+        request = context.flow.request
+
+        request_key = response.headers.get(custom_headers.MOCK_REQUEST_KEY)
+        if request_key:
+            Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Mocked{bcolors.ENDC} {request.url} -> {request_key}")
+
+        fixture_path = response.headers.get(custom_headers.MOCK_FIXTURE_PATH)
+        if fixture_path:
+            Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Mocked{bcolors.ENDC} {request.url} -> {fixture_path}")
+
     if os.environ.get(env_vars.AGENT_SIMULATE_LATENCY):
         response = context.response
         start_time = context.start_time

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/app/proxy/mock/eval_fixtures_service.py

@@ -10,6 +10,7 @@ from requests.structures import CaseInsensitiveDict
 from typing import Union
 
 from stoobly_agent.lib.logger import bcolors, Logger
+from stoobly_agent.config.constants.custom_headers import MOCK_FIXTURE_PATH
 
 from .types import Fixtures
 
@@ -20,45 +21,50 @@ class Options():
   response_fixtures: Fixtures
 
 def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Response, None]:
-  fixture_path = None
+  fixture_path = request.headers.get(MOCK_FIXTURE_PATH)
   headers = CaseInsensitiveDict()
   status_code = 200
 
-  response_fixtures = options.get('response_fixtures')
-  fixture: dict = __eval_response_fixtures(request, response_fixtures)
-
-  if not fixture:
-    public_directory_path = options.get('public_directory_path')
-
-    if not public_directory_path:
+  if fixture_path:
+    if not os.path.exists(fixture_path):
       return
+  else:
+    response_fixtures = options.get('response_fixtures')
+    fixture: dict = __eval_response_fixtures(request, response_fixtures)
 
-    request_path = 'index' if request.path == '/' else request.path
-    _fixture_path = os.path.join(public_directory_path, request_path.lstrip('/'))
-    if request.headers.get('accept'):
-      fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
+    if not fixture:
+      public_directory_path = options.get('public_directory_path')
 
-    if not fixture_path:
-      fixture_path = _fixture_path
+      if not public_directory_path:
+        return
 
-    if not os.path.isfile(fixture_path):
-      return
-  else:
-    fixture_path = fixture.get('path')
-    if not fixture_path or not os.path.isfile(fixture_path):
-      return
+      request_path = 'index' if request.path == '/' else request.path
+      _fixture_path = os.path.join(public_directory_path, request_path.lstrip('/'))
+      if request.headers.get('accept'):
+        fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
+
+      if not fixture_path:
+        fixture_path = _fixture_path
 
-    _headers = fixture.get('headers')
-    headers = CaseInsensitiveDict(_headers if isinstance(_headers, dict) else {}) 
+      if not os.path.isfile(fixture_path):
+        return
+    else:
+      fixture_path = fixture.get('path')
+      if not fixture_path or not os.path.isfile(fixture_path):
+        return
 
-    if fixture.get('status_code'):
-      status_code = fixture.get('status_code')
+      _headers = fixture.get('headers')
+      headers = CaseInsensitiveDict(_headers if isinstance(_headers, dict) else {}) 
 
+      if fixture.get('status_code'):
+        status_code = fixture.get('status_code')
+    
   with open(fixture_path, 'rb') as fp:
     response = Response()
 
     response.status_code = status_code
     response.raw = BytesIO(fp.read()) 
+    headers[MOCK_FIXTURE_PATH] = fixture_path
     response.headers = headers
 
     if not response.headers.get('content-type'):

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/config/constants/custom_headers.py

@@ -2,6 +2,7 @@ ALIAS_RESOLVE_STRATEGY = 'X-Stoobly-Alias-Resolve-Strategy'
 CONTENT_TYPE = 'X-Stoobly-Content-Type'
 CONTENT_TYPE_TEST_RESULTS = 'test/results'
 REMOTE_PROJECT_KEY = 'X-Stoobly-Endpoints-Project-Id'
+MOCK_FIXTURE_PATH = 'X-Stoobly-Fixture-Path'
 MOCK_POLICY = 'X-Stoobly-Mock-Policy'
 MOCK_REQUEST_ID = 'X-Stoobly-Request-Id'
 MOCK_REQUEST_ENDPOINT_ID = 'X-Stoobly-Request-Endpoint-Id'

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/lib/api/keys/uuid_key.py

@@ -28,6 +28,10 @@ class UuidKey(ResourceKey):
     toks = key.split(DELIMITTER)
 
     d = {}
+
+    if len(toks) == 0:
+      return d
+
     for tok in toks:
       d[tok[0]] = tok[1:]
 

{stoobly_agent-1.9.0 → stoobly_agent-1.9.2}/stoobly_agent/test/app/cli/request/request_snapshot_test.py

@@ -1,16 +1,18 @@
 import pdb
 import pytest
+import time
 
 from click.testing import CliRunner
-
-from stoobly_agent.test.test_helper import DETERMINISTIC_GET_REQUEST_URL, reset
+from typing import List
 
 from stoobly_agent.app.models.adapters.orm import JoinedRequestStringAdapter
 from stoobly_agent.app.models.factories.resource.local_db.helpers.log import Log
-from stoobly_agent.app.models.factories.resource.local_db.helpers.log_event import DELETE_ACTION
+from stoobly_agent.app.models.factories.resource.local_db.helpers.log_event import DELETE_ACTION, PUT_ACTION
 from stoobly_agent.app.models.factories.resource.local_db.helpers.request_snapshot import RequestSnapshot
-from stoobly_agent.cli import record, request
+from stoobly_agent.cli import record, request, scenario, snapshot
 from stoobly_agent.lib.orm.request import Request
+from stoobly_agent.lib.orm.scenario import Scenario
+from stoobly_agent.test.test_helper import DETERMINISTIC_GET_REQUEST_URL, NON_DETERMINISTIC_GET_REQUEST_URL, reset
 
 @pytest.fixture(scope='module')
 def runner():
@@ -122,3 +124,51 @@ class TestRequestSnapshot():
 
         event = unprocessed_events[0]
         assert event.resource_uuid == recorded_request_two.uuid
+        assert event.action == PUT_ACTION
+
+    class TestWhenDeleteFirst():
+
+      @pytest.fixture(scope='class')
+      def recorded_request(self, runner: CliRunner):
+        record_result = runner.invoke(record, [DETERMINISTIC_GET_REQUEST_URL])
+        assert record_result.exit_code == 0
+        return Request.last()
+
+      def test_initial_delete(self, runner: CliRunner, recorded_request: Request):
+        snapshot_result = runner.invoke(request, ['snapshot', '--action', DELETE_ACTION, recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        log = Log()
+
+        events = log.raw_events
+        assert len(events) == 1
+
+        unprocessed_events = log.unprocessed_events
+        assert len(unprocessed_events) == 0
+
+      def test_puts(self, runner: CliRunner, recorded_request: Request):
+        snapshot_result = runner.invoke(request, ['snapshot', '--action', PUT_ACTION, recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        log = Log()
+
+        events = log.raw_events
+        assert len(events) == 2
+
+        unprocessed_events = log.unprocessed_events
+        assert len(unprocessed_events) == 1
+        assert unprocessed_events[0].action == PUT_ACTION
+
+      def test_final_delete(self, runner: CliRunner, recorded_request: Request):
+        snapshot_result = runner.invoke(request, ['snapshot', '--action', DELETE_ACTION, recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        log = Log()
+
+        events = log.events
+        assert len(events) == 3
+
+        collapsed_events = log.collapse(events)
+        assert len(collapsed_events) == 1
+        unprocessed_events = log.unprocessed_events
+        assert len(unprocessed_events) == 0