stoobly-agent 0.34.7__py3-none-any.whl → 0.34.9__py3-none-any.whl

stoobly_agent/app/cli/helpers/schema_builder.py

@@ -16,97 +16,20 @@ class SchemaBuilder:
     self.param_column_name: str = param_column_name
 
   def build(self, params):
-    self.param_names_created = {}
-    self.__traverse('', params, None)
-    return self.param_names_created.values()
-
-  ###
-  #
-  # @param name [String] name of current param
-  # @param value [Object] value of current param
-  # @param param [QueryParamName, BodyParamName, ResponseParamName] parent param record
-  #
-  def __traverse(self, name: str, value, param: RequestComponentName):
-    if type(value) is list:
-      self.__traverse_array(name, value, param)
-    elif type(value) is dict:
-      self.__traverse_hash(name, value, param)
-
-  def __traverse_array(self, name: str, value, parent_param: RequestComponentName):
-    columns = {
-      'endpoint_id': self.endpoint_id,
-      'name':  f"{name.capitalize()}Element",
-      'query': f"{parent_param.get('query')}[*]" if parent_param else '[*]'
-    }
-    columns[self.param_column_name + '_id'] = parent_param['id'] if parent_param else None
-
-    # Iterate
-    types = {}
-
-    for e in value:
-      # Example of e = {'id': {'value': 0, 'required': False}}
-      type_value = None
-      if e.get('value') is not None:
-        type_value = e.get('value')
-      else:
-        type_value = e
-
-      _type = self.__infer_type(type_value)
-
-      if types.get(_type) is None:
-        columns['inferred_type'] = convert(_type)
-        types[_type] = self.__find_or_create_by(columns)
-    
-      self.__traverse('', type_value, types[_type])
-
-  def __traverse_hash(self, name, value, parent_param: RequestComponentName):
-    # Iterate
-    for k, v in value.items():
-      columns = {
+    params_list = []
+    for literal_param in params:
+      param: RequestComponentName = {
         'endpoint_id': self.endpoint_id,
-        'inferred_type': convert(self.__infer_type(v['value'])),
-        'is_required': v.get('required') is True,
-
+        'name': literal_param['name'],
+        'query': literal_param['query'],
+        'is_required': literal_param['required'],
+        'inferred_type': convert(self.__infer_type(literal_param['value'])),
         'is_deterministic': True,
-        'name': k,
-        'query': f"{parent_param.get('query')}.{k}" if parent_param else k, 
+        'id': literal_param['id'],
+        f"{self.param_column_name}_id": literal_param['parent_id']
       }
-      columns[self.param_column_name + '_id'] = parent_param['id'] if parent_param else None
-      param = self.__find_or_create_by(columns)
-
-      self.__traverse(k, v['value'], param)
-
-  def __find_or_create_by(self, columns):
-    param = self.__find_by(columns) 
-
-    if param is None:
-      param = self.__create(columns)
-
-    return param
-
-  def __find_by(self, columns) -> Union[RequestComponentName, None]:
-    for id, param_name in self.param_names_created.items():
-      matches = True
-
-      for key in columns:
-        if key not in param_name:
-          matches = False
-          break
-
-        if param_name[key] != columns[key]:
-          matches = False
-          break
-      
-      if matches:
-        return param_name
-
-  def __create(self, columns):
-    param: RequestComponentName = columns.copy()
-    param['id'] = len(self.param_names_created.keys()) + 1
-
-    self.param_names_created[param['id']] = param
-
-    return param
+      params_list.append(param)
+    return params_list
 
   def __infer_type(self, val) -> str:
     return str(val.__class__)

stoobly_agent/app/proxy/replay/body_parser_service.py

@@ -76,7 +76,7 @@ def parse_multipart_form_data(content, content_type) -> Dict[bytes, bytes]:
 
     params_array = []
     for ele in decoded_multipart:
-        params_array.append((decode(ele[0]), decode(ele[1])))
+        params_array.append((decode(ele[0]), ele[1]))
  
     return MultiDict(params_array)
 

stoobly_agent/app/proxy/run.py

@@ -58,7 +58,6 @@ def __get_intercept_handler_path():
 def __with_static_options(config: MitmproxyConfig, cli_options):
     options = (
         'block_global=false',
-        'flow_detail=1',
         f"scripts={__get_intercept_handler_path()}",
         'upstream_cert=false'
     )

stoobly_agent/app/proxy/utils/allowed_request_service.py

@@ -45,8 +45,18 @@ def __request_excluded(request: MitmproxyRequest, exclude_rules: List[FirewallRu
     return False
 
 def __request_included(request: MitmproxyRequest, include_rules: List[FirewallRule]):
+    if not include_rules:
+        return True
+
     method = request.method.upper()
     rules = list(filter(lambda rule: method in rule.methods, include_rules))
+
+    # If there are include rules, but none that match the request's method,
+    # then we know that none of the include rules will match the request
+    if len(include_rules) > 0 and len(rules) == 0:
+        Logger.instance().info(f"{bcolors.OKBLUE}{request.method} {request.url} not included by firewall rule{bcolors.ENDC}")
+        return False
+
     patterns = list(map(lambda rule: rule.pattern, rules))
     if not __include(request, patterns):
         Logger.instance().info(f"{bcolors.OKBLUE}{request.method} {request.url} not included by firewall rule{bcolors.ENDC}")

stoobly_agent/cli.py

@@ -86,7 +86,16 @@ def init(**kwargs):
   Passphrase for decrypting the private key provided in the --cert option. Note that passing cert_passphrase on the command line makes your passphrase visible in your system's process list. Specify it in
   config.yaml to avoid this.
 ''')
+@click.option('--confdir', default=os.path.join(os.path.expanduser('~'), '.mitmproxy'), help='Location of the default mitmproxy configuration files.')
 @click.option('--connection-strategy', help=', '.join(CONNECTION_STRATEGIES), type=click.Choice(CONNECTION_STRATEGIES))
+@click.option('--flow-detail', default='1', type=click.Choice(['1', '2', '3', '4']), help='''
+  The display detail level for flows in mitmdump: 0 (quiet) to 4 (very verbose).
+  0: no output
+  1: shortened request URL with response status code
+  2: full request URL with response status code and HTTP headers
+  3: 2 + truncated response content, content of WebSocket and TCP messages (content_view_lines_cutoff: 512)
+  4: 3 + nothing is truncated
+''')
 @click.option('--headless', is_flag=True, default=False, help='Disable starting UI.')
 @click.option('--intercept', is_flag=True, default=False, help='Enable intercept on run.')
 @click.option('--log-level', default=logger.INFO, type=click.Choice([logger.DEBUG, logger.INFO, logger.WARNING, logger.ERROR]), help='''
@@ -218,7 +227,7 @@ def __build_request_from_curl(**kwargs):
 
     if len(toks) != 2:
       continue
-    
+
     headers[toks[0].strip()] = toks[1].strip()
 
   return requests.Request(
@@ -226,4 +235,4 @@ def __build_request_from_curl(**kwargs):
     headers=headers,
     method=kwargs['request'],
     url=kwargs['url']
-  )
+  )

stoobly_agent/test/app/cli/helpers/openapi_endpoint_adapter_missing_info_test.py

@@ -61,39 +61,49 @@ class TestOpenApiEndpointAdapterMissingInfo():
       "response_param_names": [
         {
           "endpoint_id": 1,
-          "inferred_type": "Integer",
-          "is_required": True,
+          "name": "Element",
+          "query": "[*]",
+          "is_required": False,
+          "inferred_type": "Hash",
           "is_deterministic": True,
-          "name": "id",
-          "query": "id",
-          "response_param_name_id": None,
           "id": 1,
+          "response_param_name_id": None
+        },
+        {
+          "endpoint_id": 1,
+          "name": "id",
+          "query": "[*].id",
+          "is_required": True,
+          "inferred_type": "Integer",
+          "is_deterministic": True,
+          "id": 2,
+          "response_param_name_id": 1,
           "values": [
             0
           ]
         },
         {
           "endpoint_id": 1,
-          "inferred_type": "String",
+          "name": "name",
+          "query": "[*].name",
           "is_required": True,
+          "inferred_type": "String",
           "is_deterministic": True,
-          "name": "name",
-          "query": "name",
-          "response_param_name_id": None,
-          "id": 2,
+          "id": 3,
+          "response_param_name_id": 1,
           "values": [
             ""
           ]
         },
         {
           "endpoint_id": 1,
-          "inferred_type": "String",
+          "name": "tag",
+          "query": "[*].tag",
           "is_required": False,
+          "inferred_type": "String",
           "is_deterministic": True,
-          "name": "tag",
-          "query": "tag",
-          "response_param_name_id": None,
-          "id": 3
+          "id": 4,
+          "response_param_name_id": 1
         }
       ],
     }

stoobly_agent/test/app/cli/helpers/openapi_endpoint_adapter_missing_oauth2_scopes_test.py

@@ -62,42 +62,52 @@ class TestOpenApiEndpointAdapterMissingOauthScopes():
           'name': 'x-next',
         },
       ],
-        "response_param_names": [
+      "response_param_names": [
         {
           "endpoint_id": 1,
-          "inferred_type": "Integer",
-          "is_required": True,
+          "name": "Element",
+          "query": "[*]",
+          "is_required": False,
+          "inferred_type": "Hash",
           "is_deterministic": True,
-          "name": "id",
-          "query": "id",
-          "response_param_name_id": None,
           "id": 1,
+          "response_param_name_id": None
+        },
+        {
+          "endpoint_id": 1,
+          "name": "id",
+          "query": "[*].id",
+          "is_required": True,
+          "inferred_type": "Integer",
+          "is_deterministic": True,
+          "id": 2,
+          "response_param_name_id": 1,
           "values": [
             0
           ]
         },
         {
           "endpoint_id": 1,
-          "inferred_type": "String",
+          "name": "name",
+          "query": "[*].name",
           "is_required": True,
+          "inferred_type": "String",
           "is_deterministic": True,
-          "name": "name",
-          "query": "name",
-          "response_param_name_id": None,
-          "id": 2,
+          "id": 3,
+          "response_param_name_id": 1,
           "values": [
             ""
           ]
         },
         {
           "endpoint_id": 1,
-          "inferred_type": "String",
+          "name": "tag",
+          "query": "[*].tag",
           "is_required": False,
+          "inferred_type": "String",
           "is_deterministic": True,
-          "name": "tag",
-          "query": "tag",
-          "response_param_name_id": None,
-          "id": 3
+          "id": 4,
+          "response_param_name_id": 1
         }
       ],
     }

stoobly_agent/test/app/cli/helpers/openapi_endpoint_adapter_missing_servers_test.py

@@ -61,39 +61,49 @@ class TestOpenApiEndpointAdapterMissingServers():
       "response_param_names": [
         {
           "endpoint_id": 1,
-          "inferred_type": "Integer",
-          "is_required": True,
+          "name": "Element",
+          "query": "[*]",
+          "is_required": False,
+          "inferred_type": "Hash",
           "is_deterministic": True,
-          "name": "id",
-          "query": "id",
-          "response_param_name_id": None,
           "id": 1,
+          "response_param_name_id": None
+        },
+        {
+          "endpoint_id": 1,
+          "name": "id",
+          "query": "[*].id",
+          "is_required": True,
+          "inferred_type": "Integer",
+          "is_deterministic": True,
+          "id": 2,
+          "response_param_name_id": 1,
           "values": [
             0
           ]
         },
         {
           "endpoint_id": 1,
-          "inferred_type": "String",
+          "name": "name",
+          "query": "[*].name",
           "is_required": True,
+          "inferred_type": "String",
           "is_deterministic": True,
-          "name": "name",
-          "query": "name",
-          "response_param_name_id": None,
-          "id": 2,
+          "id": 3,
+          "response_param_name_id": 1,
           "values": [
             ""
           ]
         },
         {
           "endpoint_id": 1,
-          "inferred_type": "String",
+          "name": "tag",
+          "query": "[*].tag",
           "is_required": False,
+          "inferred_type": "String",
           "is_deterministic": True,
-          "name": "tag",
-          "query": "tag",
-          "response_param_name_id": None,
-          "id": 3
+          "id": 4,
+          "response_param_name_id": 1
         }
       ],
     }

stoobly_agent/test/app/cli/helpers/openapi_endpoint_adapter_test.py

@@ -73,8 +73,9 @@ class TestOpenApiEndpointAdapter():
             'endpoint_id': 1,
             'id': 2,
             'inferred_type': 'String',
+            'is_deterministic': True,
             'is_required': False,
-            'name': 'TagsElement',
+            'name': 'tagsElement',
             'query': 'tags[*]',
             'query_param_name_id': 1,
           },
@@ -104,6 +105,7 @@ class TestOpenApiEndpointAdapter():
             'endpoint_id': 1,
             'id': 1,
             'inferred_type': 'Hash',
+            'is_deterministic': True,
             'is_required': False,
             'name': 'Element',
             'query': '[*]',
@@ -365,6 +367,56 @@ class TestOpenApiEndpointAdapter():
             'query': 'apis',
             'response_param_name_id': None,
           },
+          {
+              'endpoint_id': 1,
+              'id': 3,
+              'inferred_type': 'Hash',
+              'is_deterministic': True,
+              'is_required': False,
+              'name': 'apisElement',
+              'query': 'apis[*]',
+              'response_param_name_id': 2,
+          },
+          {
+              'endpoint_id': 1,
+              'id': 4,
+              'inferred_type': 'String',
+              'is_deterministic': True,
+              'is_required': False,
+              'name': 'apiKey',
+              'query': 'apis[*].apiKey',
+              'response_param_name_id': 3,
+          },
+          {
+              'endpoint_id': 1,
+              'id': 5,
+              'inferred_type': 'String',
+              'is_deterministic': True,
+              'is_required': False,
+              'name': 'apiVersionNumber',
+              'query': 'apis[*].apiVersionNumber',
+              'response_param_name_id': 3,
+          },
+          {
+              'endpoint_id': 1,
+              'id': 6,
+              'inferred_type': 'String',
+              'is_deterministic': True,
+              'is_required': False,
+              'name': 'apiUrl',
+              'query': 'apis[*].apiUrl',
+              'response_param_name_id': 3,
+          },
+          {
+              'endpoint_id': 1,
+              'id': 7,
+              'inferred_type': 'String',
+              'is_deterministic': True,
+              'is_required': False,
+              'name': 'apiDocumentationUrl',
+              'query': 'apis[*].apiDocumentationUrl',
+              'response_param_name_id': 3,
+          },
         ],
       }
 
@@ -475,6 +527,18 @@ class TestOpenApiEndpointAdapter():
             'type': 'static',
           },
         ],
+     'response_param_names': [
+          {
+              'endpoint_id': 3,
+              'id': 1,
+              'inferred_type': 'Hash',
+              'is_deterministic': True,
+              'is_required': False,
+              'name': 'Element',
+              'query': '[*]',
+              'response_param_name_id': None,
+          },
+        ],
       }
 
     @pytest.fixture(scope='class')
@@ -485,6 +549,7 @@ class TestOpenApiEndpointAdapter():
       http_endpoint_version['body_param_names'][0]['endpoint_id'] = 6
       http_endpoint_version['body_param_names'][1]['endpoint_id'] = 6
       http_endpoint_version['body_param_names'][2]['endpoint_id'] = 6
+      http_endpoint_version['response_param_names'][0]['endpoint_id'] = 6
       return http_endpoint_version
 
     def test_adapt_from_file(self, open_api_endpoint_adapter, uspto_file_path, expected_get_root_https, expected_get_root_http, expected_get_dataset_version_fields_https, expected_get_dataset_version_fields_http, expected_post_dataset_version_records_https, expected_post_dataset_version_records_http):
@@ -694,14 +759,14 @@ class TestOpenApiEndpointAdapter():
             'values': [0]
           },
           {
-            'body_param_name_id': None,
+            'body_param_name_id': 4,
             'endpoint_id': 2,
             'id': 7,
             'inferred_type': 'Hash',
             'is_deterministic': True,
             'is_required': False,
             'name': 'adoption',
-            'query': 'adoption'
+            'query': 'extra.adoption'
           },
           {
             'body_param_name_id': 7,
@@ -711,7 +776,7 @@ class TestOpenApiEndpointAdapter():
             'is_deterministic': True,
             'is_required': True,
             'name': 'adopted',
-            'query': 'adoption.adopted',
+            'query': 'extra.adoption.adopted',
             'values': [False]
           },
           {
@@ -722,7 +787,7 @@ class TestOpenApiEndpointAdapter():
             'is_deterministic': True,
             'is_required': False,
             'name': 'shelter',
-            'query': 'adoption.shelter'
+            'query': 'extra.adoption.shelter'
           },
           {
             'body_param_name_id': None,
@@ -840,8 +905,9 @@ class TestOpenApiEndpointAdapter():
             'endpoint_id': 1,
             'id': 7,
             'inferred_type': 'String',
+            'is_deterministic': True,
             'is_required': False,
-            'name': 'PhotourlsElement',
+            'name': 'photoUrlsElement',
             'query': 'photoUrls[*]'
           },
           {
@@ -859,8 +925,9 @@ class TestOpenApiEndpointAdapter():
             'endpoint_id': 1,
             'id': 9,
             'inferred_type': 'Hash',
+            'is_deterministic': True,
             'is_required': False,
-            'name': 'TagsElement',
+            'name': 'tagsElement',
             'query': 'tags[*]'
           },
           {
@@ -971,8 +1038,9 @@ class TestOpenApiEndpointAdapter():
             'endpoint_id': 1,
             'id': 7,
             'inferred_type': 'String',
+            'is_deterministic': True,
             'is_required': False,
-            'name': 'PhotourlsElement',
+            'name': 'photoUrlsElement',
             'query': 'photoUrls[*]',
             'response_param_name_id': 6,
           },
@@ -990,8 +1058,9 @@ class TestOpenApiEndpointAdapter():
             'endpoint_id': 1,
             'id': 9,
             'inferred_type': 'Hash',
+            'is_deterministic': True,
             'is_required': False,
-            'name': 'TagsElement',
+            'name': 'tagsElement',
             'query': 'tags[*]',
             'response_param_name_id': 8,
           },
@@ -1043,6 +1112,7 @@ class TestOpenApiEndpointAdapter():
             'endpoint_id': 14,
             'id': 1,
             'inferred_type': 'Hash',
+            'is_deterministic': True,
             'is_required': False,
             'name': 'Element',
             'query': '[*]'

stoobly_agent/test/app/cli/helpers/schema_builder_test.py

@@ -7,7 +7,10 @@ class TestSchemaBuilder():
 
   def test_it_builds_single_level_dict(self):
     builder = SchemaBuilder(-1, 'query_param_name')
-    param_names = builder.build({ 'id': {'value':1}, 'name': {'value':'test'} })
+    param_names = builder.build([
+      {'name': 'id', 'value': 0, 'query': 'id', 'required': False, 'id': 1, "parent_id": None},
+      {'name': 'name', 'value': '', 'query': 'name', 'required': False, 'id': 2, "parent_id": None}
+    ])
 
     param_names_index = self.__index(param_names)
     expected_queries = ['id.Integer', 'name.String']
@@ -17,11 +20,17 @@ class TestSchemaBuilder():
 
   def test_it_builds_two_level_dict(self):
     builder = SchemaBuilder(-1, 'query_param_name')
-    param_names = builder.build({ 'list': {'value': [{'value':{ 'id': {'value':1}, 'name': {'value':'test'} }}] }, 'total': {'value':1} })
+    param_names = builder.build([
+      {'name': 'list', 'value': [], 'query': 'list', 'required': False, 'id': 1, "parent_id": None},
+      {'name': 'ListElement', 'value': {}, 'query': 'list[*]', 'required': False, 'id': 2, "parent_id": 1},
+      {'name': 'id', 'value': 0, 'query': 'list[*].id', 'required': False, 'id': 3, "parent_id": 2},
+      {'name': 'name', 'value': '', 'query': 'list[*].name', 'required': False, 'id': 4, "parent_id": 2},
+      {'name': 'total', 'value': 0, 'query': 'total', 'required': False, 'id': 5, "parent_id": None}
+    ])
 
     param_names_index = self.__index(param_names)
     expected_queries = [
-      'list.Array', 'total.Integer', 'list[*].id.Integer', 'list[*].name.String'
+      'list.Array', 'list[*].Hash', 'list[*].id.Integer', 'list[*].name.String', 'total.Integer'
     ]
 
     for query in expected_queries:
@@ -29,11 +38,13 @@ class TestSchemaBuilder():
 
   def test_it_builds_single_level_array(self):
     builder = SchemaBuilder(-1, 'body_param_name')
-    param_names = builder.build([{'value':1}, {'value':'a'}])
+    param_names = builder.build([
+      {'name': 'Element', 'value': 0, 'query': '[*]', 'required': False, 'id': 1, "parent_id": None},
+    ])
 
     param_names_index = self.__index(param_names)
     expected_queries = [
-      '[*].Integer', '[*].String'
+      '[*].Integer'
     ]
 
     for query in expected_queries:
@@ -41,11 +52,15 @@ class TestSchemaBuilder():
 
   def test_it_builds_item_and_single_level_array(self):
     builder = SchemaBuilder(-1, 'query_param_name')
-    param_names = builder.build({ 'tags': {'value':[{'value':''}]}, 'limit': {'value':0} })
+    param_names = builder.build([
+      {'name': 'tags', 'value': [], 'query': 'tags', 'required': False, 'id': 1, "parent_id": None},
+      {'name': 'TagsElement', 'value': '', 'query': 'tags[*]', 'required': False, 'id': 2, "parent_id": 1},
+      {'name': 'limit', 'value': 0, 'query': 'limit', 'required': False, 'id': 3, "parent_id": None}
+    ])
 
     param_names_index = self.__index(param_names)
     expected_queries = [
-      'limit.Integer', 'tags.Array', 'tags[*].String' 
+      'tags.Array', 'tags[*].String', 'limit.Integer' 
     ]
 
     for query in expected_queries:

stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION

@@ -1 +1 @@
-0.34.6
+0.34.8

stoobly_agent/test/app/proxy/replay/body_parser_service_test.py

@@ -16,11 +16,11 @@ class TestMultipart():
   def test_decodes_response(self, multipart_string: bytes, content_type: str):
     multidict = decode_response(multipart_string, content_type)
 
-    assert multidict.get('author') == 'John Smith'
-    assert multidict.get('file') == 'Hello World'
+    assert multidict.get('author') == b'John Smith'
+    assert multidict.get('file') == b'Hello World'
 
   def test_encodes_response(self, content_type: str):
-    expected_params = { 'author': 'John Smith', 'file': 'Hello World'}
+    expected_params = { 'author': b'John Smith', 'file': b'Hello World'}
     multipart_string = encode_response(expected_params, content_type)
 
     multidict = decode_response(multipart_string, content_type)

{stoobly_agent-0.34.7.dist-info → stoobly_agent-0.34.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: stoobly-agent
-Version: 0.34.7
+Version: 0.34.9
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le