stoobly-agent 0.21.2__py3-none-any.whl → 0.22.3__py3-none-any.whl

stoobly_agent/__init__.py

@@ -1 +1,2 @@
-VERSION = '0.21.2'
+COMMAND = 'stoobly-agent'
+VERSION = '0.22.3'

stoobly_agent/app/api/bodies_controller.py

@@ -41,12 +41,12 @@ class BodiesController:
 
         accepted_headers = ['content-encoding', 'content-length', 'content-type']
         for header, val in request.headers.items():
-            decoded_header = header.decode().lower()
+            decoded_header = header.lower()
 
             if decoded_header not in accepted_headers:
                 continue 
 
-            headers[decoded_header] = val.decode()
+            headers[decoded_header] = val
 
         context.render(
             data = request.data,

stoobly_agent/app/api/requests_controller.py

@@ -1,3 +1,4 @@
+import json
 import pdb
 import requests
 
@@ -8,15 +9,15 @@ from urllib.parse import urlparse
 from stoobly_agent.app.api.simple_http_request_handler import SimpleHTTPRequestHandler
 from stoobly_agent.app.cli.helpers.context import ReplayContext
 from stoobly_agent.app.models.adapters.joined_request_adapter import JoinedRequestAdapter
-from stoobly_agent.app.models.adapters.mitmproxy_request_adapter import MitmproxyRequestAdapter
-from stoobly_agent.app.models.adapters.mitmproxy_response_adapter import MitmproxyResponseAdapter
+from stoobly_agent.app.models.adapters.python import PythonRequestAdapterFactory, PythonResponseAdapterFactory
 from stoobly_agent.app.models.adapters.raw_http_request_adapter import RawHttpRequestAdapter
 from stoobly_agent.app.models.adapters.raw_http_response_adapter import RawHttpResponseAdapter
 from stoobly_agent.app.models.request_model import RequestModel
 from stoobly_agent.app.models.schemas.request import Request
-from stoobly_agent.app.proxy.replay.replay_request_service import replay_with_rewrite
+from stoobly_agent.app.proxy.replay.replay_request_service import replay
 from stoobly_agent.app.proxy.upload.upload_request_service import upload_staged_request
 from stoobly_agent.app.settings import Settings
+from stoobly_agent.config.constants import mode
 from stoobly_agent.lib.orm.replayed_response import ReplayedResponse
 from stoobly_agent.lib.orm.request import Request as OrmRequest
 
@@ -53,8 +54,8 @@ class RequestsController:
         request_adapter = RawHttpRequestAdapter(joined_request.request_string.get())
         response_adapter = RawHttpResponseAdapter(joined_request.response_string.get())
 
-        mitmproxy_request = MitmproxyRequestAdapter(request_adapter.protocol, request_adapter.to_request()).adapt()
-        mitmproxy_response = MitmproxyResponseAdapter(response_adapter.protocol, response_adapter.to_response()).adapt()
+        mitmproxy_request = PythonRequestAdapterFactory(request_adapter.to_request()).mitmproxy_request(request_adapter.protocol)
+        mitmproxy_response = PythonResponseAdapterFactory(response_adapter.to_response()).mitmproxy_response()
 
         class MitmproxyFlowMock():
             def __init__(self, request, response):
@@ -67,6 +68,7 @@ class RequestsController:
         request = request_model.create(**{
             'flow': mitmproxy_flow_mock,
             'joined_request': joined_request,
+            'scenario_id': body_params.get('scenario_id'),
         })
 
         if not request:
@@ -206,10 +208,17 @@ class RequestsController:
 
     # PUT /requests/send
     def send(self, context: SimpleHTTPRequestHandler):
+        headers = []
+
+        try:
+            headers = json.load(context.params.get('headers'))
+        except Exception as e:
+            pass
+
         url = urlparse(context.params.get('url'))
         request_response = {
             'body': context.params.get('body'),
-            'headers': context.params.get('headers'),
+            'headers': headers,
             'method': context.params.get('method'),
             'path': url.path,
             'password': url.password,
@@ -220,7 +229,13 @@ class RequestsController:
         }
 
         replay_context = ReplayContext(Request(request_response))
-        self.__send(context, replay_context) 
+        self.__send(context, replay_context)
+
+    def export(self, context: SimpleHTTPRequestHandler):
+        context.parse_path_params({
+            'id': 1
+        })
+        request_id = int(context.params.get('id'))
 
     def __request_model(self, context: SimpleHTTPRequestHandler):
         request_model = RequestModel(Settings.instance())
@@ -228,18 +243,21 @@ class RequestsController:
         return request_model
 
     def __replay(self, context: SimpleHTTPRequestHandler, replay_context: ReplayContext):
-        self.__send(context, replay_context, self.__create_replayed_response) 
+        save = bool(context.params.get('save'))
+        callback = self.__create_replayed_response if save else None
+
+        self.__send(context, replay_context, callback) 
 
     def __send(self, context: SimpleHTTPRequestHandler, replay_context: ReplayContext, callback = None):
         now = time()
-        res = replay_with_rewrite(replay_context)
+        res = replay(replay_context, { 'mode': mode.REPLAY })
         received_at = time()
 
         if callback:
             callback(context, res, int((received_at - now) * 1000))
 
         context.render(
-            data = res.raw.data,
+            data = res.raw.data if hasattr(res, 'raw') else res.content,
             headers = res.headers,
             status = res.status_code
         )

stoobly_agent/app/api/simple_http_request_handler.py

@@ -62,7 +62,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         headers = self.filter_headers(kwargs.get('headers'), {
             'TRANSFER-ENCODING': 'CHUNKED',
         })
-        headers['Content-Length'] = len(body)
+        headers['Content-Length'] = str(len(body))
 
         self.enable_cors(headers)
         self.render_headers(headers)

stoobly_agent/app/cli/ca_cert_installer.py

@@ -7,19 +7,22 @@ class CACertInstaller():
         home_dir = os.path.expanduser('~')
         self.mitmproxy_certs_dir = os.path.join(home_dir, '.mitmproxy')
         self.pem_file_name = 'mitmproxy-ca-cert.pem'
+        self.cer_file_name = 'mitmproxy-ca-cert.cer'
         self.crt_file_name = 'mitmproxy-ca-cert.crt'
 
+    @property
+    def mitm_crt_absolute_path(self):
+        return os.path.join(self.mitmproxy_certs_dir, self.crt_file_name)
+
     # https://askubuntu.com/a/94861
     def handle_debian(self):
         extra_ca_certs_dir = '/usr/local/share/ca-certificates/extra'
-        mitm_pem_absolute_path = os.path.join(self.mitmproxy_certs_dir, self.pem_file_name)
-        mitm_crt_absolute_path = os.path.join(self.mitmproxy_certs_dir, self.crt_file_name)
+        mitm_cer_absolute_path = os.path.join(self.mitmproxy_certs_dir, self.cer_file_name)
         extra_crt_absolute_path = os.path.join(extra_ca_certs_dir, self.crt_file_name)
 
         subprocess.run(f"sudo mkdir -p {extra_ca_certs_dir}".split(), check=True)
-        subprocess.run(f"openssl x509 -in {mitm_pem_absolute_path} -inform PEM -out {mitm_crt_absolute_path}".split(), check=True)
-        subprocess.run(f"sudo cp {mitm_crt_absolute_path} {extra_crt_absolute_path}".split(), check=True)
-        subprocess.run('sudo update-ca-certificates'.split(), check=True)
+        subprocess.run(f"sudo cp {mitm_cer_absolute_path} {extra_crt_absolute_path}".split(), check=True)
+        subprocess.run("sudo update-ca-certificates".split(), check=True)
 
     # https://www.dssw.co.uk/reference/security.html
     def handle_darwin(self):
@@ -27,3 +30,11 @@ class CACertInstaller():
         mitm_pem_absolute_path = os.path.join(self.mitmproxy_certs_dir, self.pem_file_name)
 
         subprocess.run(f"sudo security add-trusted-cert -d -p ssl -p basic -k {system_keychain_path} {mitm_pem_absolute_path}".split(), check=True)
+
+    def handle_rhel(self):
+        ca_trust_dir = '/etc/pki/ca-trust/source/anchors'
+        mitm_cer_absolute_path = os.path.join(self.mitmproxy_certs_dir, self.cer_file_name)
+        ca_trust_crt_absolute_path = os.path.join(ca_trust_dir, self.crt_file_name)
+
+        subprocess.run(f"sudo cp {mitm_cer_absolute_path} {ca_trust_crt_absolute_path}".split(), check=True)
+        subprocess.run("sudo update-ca-trust extract".split(), check=True)

stoobly_agent/app/cli/config_cli.py

@@ -223,7 +223,16 @@ if is_remote:
         pass
 
     @project.command(
-        help="Describe project"
+        help="Use local project."
+    )
+    def local(**kwargs):
+        project_key = ProjectKey.local_key
+        __set_project_key(project_key)
+
+        print("Using local project!")
+
+    @project.command(
+        help="Describe project."
     )
     @click.option('--select', multiple=True, help='Select column(s) to display.')
     @click.option('--without-headers', is_flag=True, default=False, help='Disable printing column headers.')
@@ -233,7 +242,7 @@ if is_remote:
 
         project_key = __project_key(settings)
         if project_key.is_local:
-            return
+            return print('Using local project')
 
         kwargs['project_key'] = project_key.raw
 
@@ -252,25 +261,9 @@ if is_remote:
     )
     @click.argument('project_key')
     def set(**kwargs):
-        settings = Settings.instance()
-
         project_key = kwargs['project_key']
-        validate_project_key(project_key)
-        _project_key = ProjectKey(project_key)
-
-        data_rule = settings.proxy.data.data_rules(_project_key.id)
-        scenario_key = data_rule.scenario_key
-
-        if scenario_key:
-            validate_scenario_key(scenario_key)
-            scenario_key = ScenarioKey(scenario_key)
 
-            if project_key.id != scenario_key.project_id:
-                data_rule.scenario_key = None
-                print("Current scenario does not belong to current project, unsetting current scenario.\n")
-
-        settings.proxy.intercept.project_key = project_key 
-        settings.commit()
+        __set_project_key(project_key)
 
         print("Project updated!")
 
@@ -291,4 +284,23 @@ def __select_parameter_rule(kwargs):
 def __project_key(settings):
     project_key = settings.proxy.intercept.project_key
     validate_project_key(project_key)
-    return ProjectKey(project_key)
+    return ProjectKey(project_key)
+
+def __set_project_key(project_key: str):
+    settings = Settings.instance()
+    validate_project_key(project_key)
+    _project_key = ProjectKey(project_key)
+
+    data_rule = settings.proxy.data.data_rules(_project_key.id)
+    scenario_key = data_rule.scenario_key
+
+    if scenario_key:
+        validate_scenario_key(scenario_key)
+        scenario_key = ScenarioKey(scenario_key)
+
+        if project_key.id != scenario_key.project_id:
+            data_rule.scenario_key = None
+            print("Current scenario does not belong to current project, unsetting current scenario.\n")
+
+    settings.proxy.intercept.project_key = project_key 
+    settings.commit()

stoobly_agent/app/cli/helpers/context.py

@@ -1,6 +1,8 @@
+import pdb
 import requests
 import time
 
+from stoobly_agent.app.models.adapters.python import PythonRequestAdapterFactory
 from stoobly_agent.app.models.schemas.request import Request
 
 class ReplayContext():
@@ -14,6 +16,11 @@ class ReplayContext():
 
     self.__sequence = None
 
+  @classmethod
+  def from_python_request(cls, request: requests.Request):
+    stoobly_request = PythonRequestAdapterFactory(request).stoobly_request()
+    return cls(Request(stoobly_request))
+
   @property
   def end_time(self):
     return self.__end_time

stoobly_agent/app/cli/helpers/handle_mock_service.py

@@ -0,0 +1,14 @@
+import requests
+
+from stoobly_agent.app.models.adapters.python import PythonResponseAdapterFactory
+from stoobly_agent.app.proxy.mitmproxy.response_facade import MitmproxyResponseFacade
+from stoobly_agent.app.proxy.upload.response_string import ResponseString
+
+RAW_FORMAT = 'raw'
+
+def print_raw_response(response: requests.Response):
+  mitmproxy_response = PythonResponseAdapterFactory(response).mitmproxy_response()
+  facade = MitmproxyResponseFacade(mitmproxy_response)
+  response_string = ResponseString(facade, None)
+
+  print(response_string.get().decode())

stoobly_agent/app/cli/helpers/print_service.py

@@ -14,7 +14,7 @@ def select_print_options(kwargs):
 def print_projects(projects, **kwargs):
     tabulate_print(
         projects, 
-        filter=['created_at', 'organization_id', 'project_id', 'starred', 'updated_at'],
+        filter=['created_at', 'is_deleted', 'organization_id', 'project_id', 'starred', 'updated_at'],
         headers=not kwargs.get('without_headers'),
         select=kwargs.get('select') or []
     )

stoobly_agent/app/cli/intercept.py

@@ -3,15 +3,59 @@ import pdb
 
 from stoobly_agent.app.cli.decorators.config import ConfigDecorator
 from stoobly_agent.app.settings import Settings
+from stoobly_agent.config.constants import mode
 
 settings = Settings.instance()
+mode_options = [mode.MOCK, mode.RECORD, mode.REPLAY]
+
+if settings.cli.features.remote:
+    mode_options.append(mode.TEST)
 
 @click.group(
     epilog="Run 'stoobly-agent intercept COMMAND --help' for more information on a command.",
-    help="Toggle request intercept"
+    help="Manage request intercept"
 )
 @click.pass_context
 def intercept(ctx):
     pass
 
-ConfigDecorator(intercept, settings, 'proxy.intercept.active').decorate()
+@intercept.command(
+    help="Enable intercept"
+)
+@click.option('--mode', type=click.Choice(mode_options))
+def enable(**kwargs):
+    settings = Settings.instance()
+
+    if kwargs['mode']:
+        settings.proxy.intercept.mode = kwargs['mode']
+
+    settings.proxy.intercept.active = True
+
+    settings.commit()
+
+    print("Intercept enabled!")
+
+@intercept.command(
+    help="Disable intercept"
+)
+def disable(**kwargs):
+    settings = Settings.instance()
+
+    settings.proxy.intercept.active = False
+
+    settings.commit()
+
+    print("Intercept disabled!")
+
+@intercept.command(
+    help="Show intercept"
+)
+def show(**kwargs):
+    settings = Settings.instance()
+
+    mode = settings.proxy.intercept.mode
+
+    if not mode:
+        print('No intercept mode set')
+    else:
+        print(f"{mode.capitalize()} {'enabled' if settings.proxy.intercept.active else 'disabled'}")

stoobly_agent/app/cli/main_group.py

@@ -26,7 +26,7 @@ class MainGroup(click.Group):
     command_groups: List[CommandGroup] = [
       {
         'name': 'Commands',
-        'commands': ['dev-tools', 'exec', 'feature', 'init'],
+        'commands': ['dev-tools', 'exec', 'feature', 'init', 'mock', 'record'],
       },
       {
         'name': 'Proxy Commands',
@@ -36,12 +36,12 @@ class MainGroup(click.Group):
 
     if self.__settings.cli.features.remote:
       command_groups.append({
-        'name': 'Remote Commands',
+        'name': 'Remote Resource Commands',
         'commands': ['project', 'report', 'request', 'scenario', 'trace'],
       })
     else:
       command_groups.append({
-        'name': 'Local Commands',
+        'name': 'Local Resource Commands',
         'commands': ['request', 'scenario'],
       })
 

stoobly_agent/app/cli/project_cli.py

@@ -57,7 +57,9 @@ def list(**kwargs):
     if not organization_key:
         try:
             project_key = ProjectKey(settings.proxy.intercept.project_key)
-            organization_key = OrganizationKey.encode(project_key.organization_id)
+
+            if not project_key.is_local:
+                organization_key = OrganizationKey.encode(project_key.organization_id)
         except InvalidProjectKey as e:
             pass
 

stoobly_agent/app/cli/request_cli.py

@@ -38,7 +38,7 @@ def request(ctx):
 )
 @click.option('--page', default=0)
 @ConditionalDecorator(lambda f: click.option('--project-key')(f), is_remote)
-@ConditionalDecorator(lambda f: click.option('--scenario-key')(f), is_remote)
+@click.option('--scenario-key')
 @click.option('--select', multiple=True, help='Select column(s) to display.')
 @click.option('--sort-by', default='created_at', help='created_at|path')
 @click.option('--sort-order', default='desc', help='asc | desc')
@@ -86,7 +86,7 @@ def list(**kwargs):
   '''
 )
 @click.option('--record', is_flag=True, default=False, help='Replay and record request.')
-@ConditionalDecorator(lambda f: click.option('--scenario-key', help='Record to scenario.')(f), is_remote)
+@click.option('--scenario-key', help='Record to scenario.')
 @click.option('--scheme', help='Rewrite request scheme.')
 @ConditionalDecorator(lambda f: click.option('--trace-id', help='Use existing trace.')(f), is_remote)
 @ConditionalDecorator(lambda f: click.option('--validate', multiple=True, help='Validate one or more aliases. Format: <NAME>=?<TYPE>')(f), is_remote)
@@ -203,9 +203,11 @@ def get(**kwargs):
 
   print(res.content)
 
-@response.command()
+@response.command(
+  help="Query properties in response"
+)
 @click.option('--query', required=True)
-@click.argument('request-key')
+@click.argument('request_key')
 def query(**kwargs):
   validate_request_key(kwargs['request_key'])
 
@@ -218,12 +220,6 @@ def query(**kwargs):
 
   print(jmespath.search(kwargs['query'], decoded_response))
 
-@response.command(
-  help="Set new mocked response"
-)
-def set(**kwargs):
-  print("Not yet implemented. Stay tuned!")
-
 request.add_command(response)
 
 def __replay(handler, kwargs) -> requests.Response:

stoobly_agent/app/models/adapters/mitmproxy/__init__.py

@@ -0,0 +1,2 @@
+from .request import MitmproxyRequestAdapterFactory
+from .response import MitmproxyResponseAdapterFactory

stoobly_agent/app/models/adapters/mitmproxy/request/__init__.py

@@ -0,0 +1,12 @@
+import pdb
+import requests
+
+from .python_adapter import PythonRequestAdapter
+
+class MitmproxyRequestAdapterFactory():
+
+  def __init__(self, request: requests.Request):
+    self.__request = request
+
+  def python_request(self):
+    return PythonRequestAdapter(self.__request).adapt()

stoobly_agent/app/models/adapters/mitmproxy/request/python_adapter.py

@@ -0,0 +1,16 @@
+import requests
+
+from mitmproxy.http import Request as MitmproxyRequest
+
+class PythonRequestAdapter():
+  
+  def __init__(self, request: MitmproxyRequest):
+    self.__request = request
+
+  def adapt(self):
+    return requests.Request(
+      method=self.__request.method,
+      url=self.__request.url,
+      headers=self.__request.headers,
+      data=self.__request.content
+    )

stoobly_agent/app/models/adapters/mitmproxy/response/__init__.py

@@ -0,0 +1,12 @@
+import pdb
+import requests
+
+from .python_adapter import PythonResponseAdapter
+
+class MitmproxyResponseAdapterFactory():
+
+  def __init__(self, response: requests.Response):
+    self.__response = response
+
+  def python_response(self):
+    return PythonResponseAdapter(self.__response).adapt()

stoobly_agent/app/models/adapters/mitmproxy/response/python_adapter.py

@@ -0,0 +1,24 @@
+from io import BytesIO
+import requests
+
+from mitmproxy.http import Response as MitmproxyResponse
+from urllib3 import HTTPResponse
+
+class PythonResponseAdapter():
+
+  def __init__(self, response: MitmproxyResponse):
+    self.__response = response
+
+  def adapt(self) -> requests.Response:
+    response = requests.Response()
+    response.status_code = self.__response.status_code
+    response.headers = self.__response.headers
+
+    response.raw = HTTPResponse(
+      body=BytesIO(self.__response.raw_content),
+      decode_content=False,
+      headers=self.__response.headers,
+      preload_content=False
+    )
+
+    return response

stoobly_agent/app/models/adapters/python/__init__.py

@@ -0,0 +1,2 @@
+from .request import PythonRequestAdapterFactory
+from .response import PythonResponseAdapterFactory

stoobly_agent/app/models/adapters/python/request/__init__.py

@@ -0,0 +1,18 @@
+import pdb
+import requests
+
+from urllib.parse import parse_qs, urlparse
+
+from .mitmproxy_adapter import MitmproxyRequestAdapter
+from .stoobly_adapter import StooblyRequestAdapter
+
+class PythonRequestAdapterFactory():
+
+  def __init__(self, request: requests.Request):
+    self.__request = request
+
+  def mitmproxy_request(self, http_version: str = 'HTTP/1.1'):
+    return MitmproxyRequestAdapter(http_version, self.__request).adapt()
+
+  def stoobly_request(self):
+    return StooblyRequestAdapter(self.__request).adapt()

stoobly_agent/app/models/adapters/python/request/mitmproxy_adapter.py

@@ -0,0 +1,53 @@
+import pdb
+import requests
+
+from mitmproxy.http import Headers, Request as MitmproxyRequest
+from urllib.parse import urlparse
+
+class MitmproxyRequestAdapter():
+
+  def __init__(self, http_version, request: requests.Request):
+    self.__http_version = http_version
+    self.__request = request
+
+  @property
+  def url(self):
+    parsed_url = urlparse(self.__request.url)
+
+    if not parsed_url.netloc:
+      parsed_url = parsed_url._replace(netloc=parsed_url.path, path='')
+
+    if not parsed_url.scheme:
+      parsed_url = parsed_url._replace(scheme='https')
+
+    return parsed_url.geturl()
+
+  @property
+  def headers(self):
+    return Headers(**self.__decode_dict(self.__request.headers))
+
+  @property
+  def data(self):
+    _data = self.__request.data
+
+    # If no data is provided to python requests.Request, it is sent to []
+    if _data == []:
+      return b''
+
+    return _data
+
+  def adapt(self):
+    request = MitmproxyRequest.make(
+      self.__request.method,
+      self.url,
+      self.data,
+      self.headers,
+    )
+    request.http_version = self.__http_version
+    return request
+
+  def __decode_dict(self, d):
+    new_d = {}
+    for k, v in d.items():
+      new_d[k.decode() if isinstance(k, bytes) else k] = v.decode() if isinstance(v, bytes) else v
+    return new_d

stoobly_agent/app/models/adapters/python/request/stoobly_adapter.py

@@ -0,0 +1,49 @@
+import pdb
+import requests
+
+from urllib.parse import parse_qs, urlparse
+
+class StooblyRequestAdapter():
+
+  def __init__(self, request: requests.Request):
+    self.__request = request
+
+  def adapt(self):
+    parsed_url = urlparse(self.__request.url)
+
+    return {
+      'body': self.__request.data,
+      'headers': self.adapt_headers(),
+      'method': self.__request.method,
+      'path': parsed_url.path,
+      'query_params': self.adapt_query_params(parsed_url.query),
+      'url': self.__request.url,
+    }
+
+  def adapt_query_params(self, query = None):
+    if not query:
+      parsed_url = urlparse(self.__request.url)
+      query = parsed_url.query
+
+    _query_params = parse_qs(query)
+
+    query_params = []
+    for k, v in _query_params.items():
+      for param in v:
+        query_params.append({
+          'name': k,
+          'value': param,
+        })
+
+    return query_params
+
+  def adapt_headers(self):
+    headers = []
+
+    for k, v in self.__request.headers.items():
+      headers.append({
+        'name': k,
+        'value': v
+      })
+
+    return headers

stoobly_agent/app/models/adapters/python/response/__init__.py

@@ -0,0 +1,13 @@
+import http
+import pdb
+import requests
+
+from .mitmproxy_adapter import MitmproxyResponseAdapter
+
+class PythonResponseAdapterFactory():
+
+  def __init__(self, response: requests.Response):
+    self.__response = response
+
+  def mitmproxy_response(self):
+    return MitmproxyResponseAdapter(self.__response).adapt()