stoobly-agent 0.34.5__tar.gz → 0.34.7__tar.gz

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: stoobly-agent
-Version: 0.34.5
+Version: 0.34.7
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/pyproject.toml

@@ -10,7 +10,7 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "0.34.5"
+version = "0.34.7"
 
 [tool.poetry.dependencies]
 python = "^3.8"

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '0.34.5'
+VERSION = '0.34.7'

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/app/cli/request_cli.py

@@ -88,7 +88,7 @@ def list(**kwargs):
 def replay(**kwargs):
   replay_handler(kwargs)
 
-if not is_remote:
+if is_local:
   @request.command(
     help="Snapshot a request"
   )

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/app/cli/snapshot_cli.py

@@ -74,6 +74,14 @@ def _list(**kwargs):
     if len(formatted_events):
       print_snapshots(formatted_events, **print_options)
 
+@snapshot.command(
+  help="Prune deleted snapshots."
+)
+@click.option('--dry-run', is_flag=True, default=False)
+def prune(**kwargs):
+  log = Log()
+  log.prune(kwargs['dry_run'])
+
 @snapshot.command(
   help="Update snapshot",
 )

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/app/models/factories/resource/local_db/helpers/log.py

@@ -5,8 +5,10 @@ import time
 from typing import List
 
 from stoobly_agent.config.data_dir import DataDir
+from stoobly_agent.lib.logger import bcolors, Logger
 
-from .log_event import LogEvent, Resource
+from .log_event import LogEvent
+from .snapshot_types import DELETE_ACTION, PUT_ACTION, Resource
 
 EVENT_DELIMITTER = "\n"
 
@@ -28,7 +30,7 @@ class Log():
     if events_count == 0:
       return [] 
 
-    return list(map(lambda raw_event: LogEvent(raw_event), self.raw_events))
+    return list(map(lambda raw_event: LogEvent(raw_event), events))
 
   @property
   def history_dir_path(self):
@@ -56,15 +58,12 @@ class Log():
   @property
   def target_events(self):
     events = self.events
-    return self.prune(events)
+    return self.collapse(events)
 
   @property
   def raw_events(self):
     contents = self.read()
-    if not contents:
-      return []
-    events = contents.strip().split(EVENT_DELIMITTER)
-    return list(filter(lambda e: not not e, events))
+    return self.build_raw_events(contents) 
 
   @property
   def unprocessed_events(self) -> List[LogEvent]:
@@ -75,14 +74,11 @@ class Log():
       return []
 
     version = self.version.strip()
-    if not version:
-      return self.prune(events)
-
-    # Find diverge point
-    version_uuids = version.split(EVENT_DELIMITTER)
-    unprocessed_events: List[LogEvent] = self.remove_processed_events(events, version_uuids)
+    version_uuids = []
+    if version:
+      version_uuids = version.split(EVENT_DELIMITTER)
 
-    return self.prune(unprocessed_events)
+    return self.remove_processed_events(events, version_uuids)
 
   @property
   def version(self):
@@ -126,6 +122,9 @@ class Log():
     with open(self.log_file_path, 'w') as fp:
       fp.write(_events + EVENT_DELIMITTER)
 
+  def build_log_events(self, raw_events) -> List[LogEvent]:
+    return list(map(lambda raw_event: LogEvent(raw_event), raw_events))
+
   def next_version(self, last_processed_uuid: str = None):
     uuids = self.uuids()
 
@@ -138,7 +137,7 @@ class Log():
 
     return self.generate_version(uuids)
 
-  def prune(self, events: List[LogEvent]) -> List[LogEvent]:
+  def collapse(self, events: List[LogEvent]) -> List[LogEvent]:
     events_count = {}
 
     # More recent events take precedence over earlier ones, keep only the most recent event 
@@ -164,6 +163,62 @@ class Log():
     serialized_event = LogEvent.serialize_put(resource)
     self.append(serialized_event)
 
+  def prune(self, dry_run = False):
+    # event uuid => history path
+    events: List[LogEvent] = []
+    path_index = {}
+
+    history_files = self.history_files
+    for file_path in history_files:
+      with open(file_path, 'r') as fp:
+        contents = fp.read().strip()
+        raw_events = self.build_raw_events(contents)
+        history_events = self.build_log_events(raw_events)
+
+        for event in history_events:
+          path_index[event.uuid] = file_path
+
+        events += history_events
+    
+    # resource_uuid => event
+    resource_index = {}
+    for event in events:
+      if event.resource_uuid not in resource_index:
+        resource_index[event.resource_uuid] = []
+
+      resource_index[event.resource_uuid].append(event)
+
+    pruned_events = self.collapse(events)
+    for event in pruned_events:
+      snapshot = event.snapshot()
+      snapshot_exists = snapshot.exists
+
+      if event.action == DELETE_ACTION or not snapshot_exists:
+        Logger.instance().info(f"{bcolors.OKBLUE}Removing {event.resource} {event.resource_uuid}{bcolors.ENDC}")
+
+        resource_events: List[LogEvent] = resource_index[event.resource_uuid]
+        removed_events = {}
+
+        for event in resource_events:
+          history_path = path_index[event.uuid]
+          if event.uuid in removed_events:
+            continue
+
+          Logger.instance().info(f"Removing event {event.uuid}")
+          self.remove_event_history(event, history_path, dry_run)
+          removed_events[event.uuid] = True
+
+        if event.action == DELETE_ACTION and snapshot_exists: 
+          if not dry_run:
+            snapshot.remove()
+          Logger.instance().info(f"Removing {event.resource} snapshot")
+
+  def build_raw_events(self, contents: str) -> List[str]:
+    if not contents:
+      return []
+    events = contents.strip().split(EVENT_DELIMITTER)
+    return list(filter(lambda e: not not e, events))
+      
   def read(self):
     history_files = self.history_files 
 
@@ -173,18 +228,56 @@ class Log():
         log.append(fp.read().strip())
 
     return EVENT_DELIMITTER.join(log)
-
-  def remove_processed_events(self, events: List[LogEvent], version_uuids: List[str]):
-    iterations = min(len(events), len(version_uuids))
-    for i in range(0, iterations):
-      event = events[i] 
-
-      if event.uuid != version_uuids[i]:
-        i -= 1
-        break
-
-    remaining_version_uuids = version_uuids[i + 1:]
-    remaining_events = events[i + 1:]
+  
+  def remove_dangling_events(self, processed_events: List[LogEvent], unprocessed_events: List[LogEvent]):
+    '''
+    Remove DELETE events where the last processed event was a PUT
+    '''
+    index = {}
+    for event in processed_events:
+      if event.action == PUT_ACTION:
+        index[event.resource_uuid] = event
+      elif event.action == DELETE_ACTION:
+        if event.resource_uuid in index:
+          del index[event.resource_uuid]
+    
+    return list(
+      filter(
+        lambda e: e.action != DELETE_ACTION or (e.action == DELETE_ACTION and e.resource_uuid in index), 
+        unprocessed_events
+      )
+    )
+
+  def remove_event_history(self, event: LogEvent, history_path: str, dry_run = False):
+    events = []
+    raw_events = []
+
+    with open(history_path, 'r') as fp:
+      contents = fp.read().strip()
+      raw_events = self.build_raw_events(contents)
+      events = self.build_log_events(raw_events)
+      events = list(filter(lambda log_event: log_event.uuid != event.uuid, events))
+
+    if len(events) == 0:
+      Logger.instance().info(f"Removing {history_path}")
+
+      if not dry_run:
+        os.remove(history_path)
+    else:
+      new_raw_events = list(map(lambda event: str(event), events))
+      Logger.instance().info(f"Updating {history_path}, Events: {len(raw_events)} -> {len(new_raw_events)}")
+
+      if not dry_run:
+        with open(history_path, 'w') as fp:
+          fp.write(EVENT_DELIMITTER.join(new_raw_events))
+
+  def remove_processed_events(self, events, version_uuids):
+    i = self.__diverge_point(events, version_uuids)
+    processed_events = self.collapse(events[0:i])
+    unprocessed_events = self.collapse(events[i:])
+    remaining_version_uuids = version_uuids[i:]
+
+    remaining_events = self.remove_dangling_events(processed_events, unprocessed_events)
     return list(filter(lambda e: e.uuid not in remaining_version_uuids, remaining_events))
 
   # Rotate log to history
@@ -201,6 +294,15 @@ class Log():
     _events = events or self.events
     return list(map(lambda e: e.uuid, _events))
 
+  def __diverge_point(self, events: List[LogEvent], version_uuids: List[str]):
+    iterations = min(len(events), len(version_uuids))
+    for i in range(0, iterations):
+      event = events[i] 
+
+      if event.uuid != version_uuids[i]:
+        return i
+    return iterations
+
   def __history_file_path(self, bucket_interval: int):
     file_name = f"{int(time.time() / bucket_interval) * bucket_interval}"
     return os.path.join(self.history_dir_path, file_name)

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/app/models/factories/resource/local_db/helpers/log_event.py

@@ -2,7 +2,7 @@ import pdb
 import uuid
 import time
 
-from typing import Callable, Literal, TypedDict
+from typing import Callable, TypedDict
 
 from stoobly_agent.lib.orm.request import Request
 from stoobly_agent.lib.orm.scenario import Scenario

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/app/models/factories/resource/local_db/helpers/request_snapshot.py

@@ -19,6 +19,10 @@ class RequestSnapshot(Snapshot):
   def backup(self):
     return self.__backup
 
+  @property
+  def exists(self):
+    return os.path.exists(self.path)
+
   @property
   def request(self):
     request_file_path = self.path

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/app/models/factories/resource/local_db/helpers/scenario_snapshot.py

@@ -20,6 +20,10 @@ class ScenarioSnapshot(Snapshot):
     self.__metadata_backup = None
     self.__requests_backup = None
 
+  @property
+  def exists(self):
+    return os.path.exists(self.requests_path) and os.path.exists(self.metadata_path)
+
   @property
   def metadata(self):
     if not os.path.exists(self.metadata_path):
@@ -84,6 +88,10 @@ class ScenarioSnapshot(Snapshot):
           request_snapshot = RequestSnapshot(uuid)
           handler(request_snapshot)
 
+  def remove(self):
+    self.remove_metadata()
+    self.remove_requests()
+
   def remove_metadata(self):
     metadata_file_path = self.metadata_path
 

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/app/proxy/run.py

@@ -124,6 +124,9 @@ def __filter_options(options):
     if 'lifecycle_hooks_path' in options:
         del options['lifecycle_hooks_path']
 
+    if 'proxyless' in options:
+        del options['proxyless']
+
     if 'public_directory_path' in options:
         del options['public_directory_path']
 

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/cli.py

@@ -98,6 +98,7 @@ def init(**kwargs):
   An empty header-value removes existing header-name headers. May be passed multiple times.
 ''')
 @click.option('--proxy-host', default='0.0.0.0', help='Address to bind proxy to.')
+@click.option('--proxyless', is_flag=True, default=False, help='Disable starting proxy.')
 @click.option('--proxy-mode', default="regular", help='''
   Proxy mode can be "regular", "transparent", "socks5",
   "reverse:SPEC", or "upstream:SPEC". For reverse and
@@ -136,7 +137,8 @@ def run(**kwargs):
     if 'headless' in kwargs and not kwargs['headless']:
         run_api(**kwargs)
 
-    run_proxy(**kwargs)
+    if 'proxyless' in kwargs and not kwargs['proxyless']:
+        run_proxy(**kwargs)
 
 @main.command(
   help="Mock request"

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/test/app/cli/request/request_snapshot_test.py

@@ -77,7 +77,7 @@ class TestRequestSnapshot():
         assert len(events) == 3
 
         unprocessed_events = log.unprocessed_events
-        assert len(unprocessed_events) == 1
+        assert len(unprocessed_events) == 0
 
     class TestWhenAppending():
 
@@ -118,7 +118,7 @@ class TestRequestSnapshot():
         assert len(events) == 3
 
         unprocessed_events = log.unprocessed_events
-        assert len(unprocessed_events) == 2
+        assert len(unprocessed_events) == 1
 
         event = unprocessed_events[0]
         assert event.resource_uuid == recorded_request_two.uuid

{stoobly_agent-0.34.5 → stoobly_agent-0.34.7}/stoobly_agent/test/app/cli/snapshot/snapshot_apply_test.py

@@ -1,5 +1,6 @@
 import pdb
 import pytest
+import time
 
 from click.testing import CliRunner
 from typing import List
@@ -23,6 +24,10 @@ class TestApply():
 
     class TestWhenDeletingRequest():
 
+      @pytest.fixture(scope='class')
+      def log(self):
+        return Log()
+
       @pytest.fixture(scope='class')
       def recorded_request(self, runner: CliRunner):
         record_result = runner.invoke(record, [DETERMINISTIC_GET_REQUEST_URL])
@@ -30,13 +35,24 @@ class TestApply():
         return Request.last()
 
       @pytest.fixture(autouse=True, scope='class')
-      def delete_event(self, runner: CliRunner, recorded_request: Request):
-        snapshot_result = runner.invoke(request, ['snapshot', '--action', DELETE_ACTION, recorded_request.key()])
+      def events(self, runner: CliRunner, recorded_request: Request, log: Log):
+        snapshot_result = runner.invoke(request, ['snapshot', recorded_request.key()])
         assert snapshot_result.exit_code == 0
 
-        log = Log()
         events = log.events
-        return events[len(events) - 1]
+        event: LogEvent = events[len(events) - 1]
+        log.version = event.uuid
+
+        time.sleep(1)
+
+        snapshot_result = runner.invoke(request, ['snapshot', '--action', DELETE_ACTION, recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        return log.events
+
+      @pytest.fixture(scope='class')
+      def delete_event(self, events: List[LogEvent]):
+        return events[1]
 
       def test_it_deletes(self, runner: CliRunner, recorded_request: Request, delete_event: LogEvent):
         apply_result = runner.invoke(snapshot, ['apply', '--force', delete_event.uuid])
@@ -82,6 +98,10 @@ class TestApply():
 
     class TestWhenDeletingScenario():
 
+      @pytest.fixture(scope='class')
+      def log(self):
+        return Log()
+
       @pytest.fixture(scope='class')
       def created_scenario(self, runner: CliRunner):
           create_result = runner.invoke(scenario, ['create', 'test'])
@@ -89,13 +109,24 @@ class TestApply():
           return Scenario.last()
 
       @pytest.fixture(autouse=True, scope='class')
-      def delete_event(self, runner: CliRunner, created_scenario: Scenario):
-        snapshot_result = runner.invoke(scenario, ['snapshot', '--action', DELETE_ACTION, created_scenario.key()])
+      def events(self, runner: CliRunner, created_scenario: Scenario, log: Log):
+        snapshot_result = runner.invoke(scenario, ['snapshot',  created_scenario.key()])
         assert snapshot_result.exit_code == 0
 
-        log = Log()
         events = log.events
-        return events[len(events) - 1]
+        event: LogEvent = events[len(events) - 1]
+        log.version = event.uuid
+
+        time.sleep(1)
+
+        snapshot_result = runner.invoke(scenario, ['snapshot', '--action', DELETE_ACTION, created_scenario.key()])
+        assert snapshot_result.exit_code == 0
+
+        return log.events
+
+      @pytest.fixture(scope='class')
+      def delete_event(self, events):
+        return events[1]
 
       def test_it_deletes(self, runner: CliRunner, created_scenario: Scenario, delete_event: LogEvent):
         snapshot_result = runner.invoke(snapshot, ['apply', '--force', delete_event.uuid])

stoobly_agent-0.34.7/stoobly_agent/test/app/cli/snapshot/snapshot_prune_test.py

@@ -0,0 +1,113 @@
+import pdb
+import pytest
+import time
+
+from click.testing import CliRunner
+from typing import List
+
+from stoobly_agent.app.models.factories.resource.local_db.helpers.log import Log
+from stoobly_agent.app.models.factories.resource.local_db.helpers.log_event import DELETE_ACTION, LogEvent
+from stoobly_agent.app.models.factories.resource.local_db.helpers.scenario_snapshot import ScenarioSnapshot
+from stoobly_agent.cli import record, request, scenario, snapshot
+from stoobly_agent.lib.orm.request import Request
+from stoobly_agent.lib.orm.scenario import Scenario
+from stoobly_agent.test.test_helper import assert_orm_request_equivalent, DETERMINISTIC_GET_REQUEST_URL, NON_DETERMINISTIC_GET_REQUEST_URL, reset
+
+@pytest.fixture(scope='module')
+def runner():
+    return CliRunner()
+
+class TestPrune():
+    @pytest.fixture(scope='class', autouse=True)
+    def settings(self):
+      return reset()
+
+    class TestWhenDeletingRequest():
+
+      @pytest.fixture(scope='class')
+      def log(self):
+        return Log()
+
+      @pytest.fixture(scope='class')
+      def recorded_request(self, runner: CliRunner):
+        record_result = runner.invoke(record, [DETERMINISTIC_GET_REQUEST_URL])
+        assert record_result.exit_code == 0
+        return Request.last()
+
+      @pytest.fixture(autouse=True, scope='class')
+      def events(self, runner: CliRunner, recorded_request: Request, log: Log):
+        snapshot_result = runner.invoke(request, ['snapshot', recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        events = log.events
+        event: LogEvent = events[len(events) - 1]
+        log.version = event.uuid
+
+        time.sleep(1)
+
+        snapshot_result = runner.invoke(request, ['snapshot', '--action', DELETE_ACTION, recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        return log.events
+
+      def test_it_prunes(self, runner: CliRunner, log: Log):
+        assert len(log.events) == 2
+        apply_result = runner.invoke(snapshot, ['prune'])
+        assert apply_result.exit_code == 0
+        assert len(log.events) == 0
+
+    class TestWhenCreatingRequest():
+
+      @pytest.fixture(scope='class')
+      def log(self):
+        return Log()
+
+      @pytest.fixture(scope='function')
+      def recorded_request(self, runner: CliRunner):
+        record_result = runner.invoke(record, [DETERMINISTIC_GET_REQUEST_URL])
+        assert record_result.exit_code == 0
+        return Request.last()
+
+      @pytest.fixture(autouse=True, scope='function')
+      def put_event(self, runner: CliRunner, recorded_request: Request):
+        snapshot_result = runner.invoke(request, ['snapshot', recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        log = Log()
+        events = log.events
+        return events[len(events) - 1]
+
+      def test_it_does_not_prune(self, runner: CliRunner, log: Log):
+        apply_result = runner.invoke(snapshot, ['prune'])
+        assert apply_result.exit_code == 0
+        assert len(log.events) == 1
+
+    class TestWhenMissingSnapshot():
+
+      @pytest.fixture(scope='class')
+      def log(self):
+        return Log()
+
+      @pytest.fixture(scope='function')
+      def recorded_request(self, runner: CliRunner):
+        record_result = runner.invoke(record, [DETERMINISTIC_GET_REQUEST_URL])
+        assert record_result.exit_code == 0
+        return Request.last()
+
+      @pytest.fixture(autouse=True, scope='function')
+      def put_event(self, runner: CliRunner, recorded_request: Request):
+        snapshot_result = runner.invoke(request, ['snapshot', recorded_request.key()])
+        assert snapshot_result.exit_code == 0
+
+        log = Log()
+        events = log.events
+        return events[len(events) - 1]
+
+      def test_it_prunes(self, runner: CliRunner, log: Log):
+        event = log.events[0]
+        event.snapshot().remove()
+        assert len(log.events) == 1
+
+        apply_result = runner.invoke(snapshot, ['prune'])
+        assert apply_result.exit_code == 0
+        assert len(log.events) == 0

stoobly_agent-0.34.7/stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION

@@ -0,0 +1 @@
+0.34.6

stoobly_agent-0.34.5/stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION

@@ -1 +0,0 @@
-0.34.3