http-content-parser 0.0.21__tar.gz → 0.0.23__tar.gz

{http_content_parser-0.0.21 → http_content_parser-0.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: http_content_parser
-Version: 0.0.21
+Version: 0.0.23
 Summary: parse http's payload and response
 Author-email: leo <suleiabc@gmail.com>
 License-File: LICENSE

http_content_parser-0.0.23/api.yaml

@@ -0,0 +1,54 @@
+v1_hfw_get_operators_get:
+  body: {}
+  header: '{"x-tt-env": "boe_env_self_testing_newfeat"}'
+  method: get
+  original_url: nacmgt-boe.bytedance.net/v1/hfw/get_operators
+  path: v1/hfw/get_operators
+  path_param: {}
+  query_param: {}
+  response: {}
+v1_hfw_get_operators_get:
+  body: {}
+  header: '{"x-tt-env": "boe_env_self_testing_newfeat"}'
+  method: get
+  original_url: nacmgt-boe.bytedance.net/v1/hfw/get_operators
+  path: v1/hfw/get_operators
+  path_param: {}
+  query_param: {}
+  response: {}
+v1_hfw_get_operators_get:
+  body: {}
+  header: '{"x-tt-env": "boe_env_self_testing_newfeat"}'
+  method: get
+  original_url: nacmgt-boe.bytedance.net/v1/hfw/get_operators
+  path: v1/hfw/get_operators
+  path_param: {}
+  query_param: {}
+  response: {}
+v1_hfw_get_operators_get:
+  body: {}
+  header: '{"x-tt-env": "boe_env_self_testing_newfeat"}'
+  method: get
+  original_url: nacmgt-boe.bytedance.net/v1/hfw/get_operators
+  path: v1/hfw/get_operators
+  path_param: {}
+  query_param: {}
+  response: {}
+v1_hfw_get_operators_get:
+  body: {}
+  header: '{"x-tt-env": "boe_env_self_testing_newfeat"}'
+  method: get
+  original_url: nacmgt-boe.bytedance.net/v1/hfw/get_operators
+  path: v1/hfw/get_operators
+  path_param: {}
+  query_param: {}
+  response: {}
+v1_hfw_get_operators_get:
+  body: {}
+  header: '{"x-tt-env": "boe_env_self_testing_newfeat"}'
+  method: get
+  original_url: nacmgt-boe.bytedance.net/v1/hfw/get_operators
+  path: v1/hfw/get_operators
+  path_param: {}
+  query_param: {}
+  response: {}

{http_content_parser-0.0.21 → http_content_parser-0.0.23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "http_content_parser"
-version = "0.0.21"
+version = "0.0.23"
 authors = [{ name = "leo", email = "suleiabc@gmail.com" }]
 description = "parse http's payload and response"
 readme = "README.md"

{http_content_parser-0.0.21 → http_content_parser-0.0.23}/src/http_content_parser/api_parser.py

@@ -34,18 +34,36 @@ class ApiModelParser:
         new_payload_list = self.__handle_duplicate_api_label(payload_list)
         return new_payload_list
 
+    def get_api_list_for_postman(self, json_dict: dict) -> list[dict]:
+        payload_list = self.convert_postman_to_list(postman_dict=json_dict)
+        # handle duplicate key
+        new_payload_list = self.__handle_duplicate_api_label_for_dict(payload_list)
+        return new_payload_list
+
     def get_api_model_for_swagger(self, json_dict: dict) -> list[ReqData]:
         payload_list = self.convert_swagger_to_model(swagger2_dict=json_dict)
         # handle duplicate key
         new_payload_list = self.__handle_duplicate_api_label(payload_list)
         return new_payload_list
 
+    def get_api_list_for_swagger(self, json_dict: dict) -> list[dict]:
+        payload_list = self.convert_swagger_to_list(swagger2_dict=json_dict)
+        # handle duplicate key
+        new_payload_list = self.__handle_duplicate_api_label_for_dict(payload_list)
+        return new_payload_list
+
     def get_api_model_for_openapi(self, json_dict: dict) -> list[ReqData]:
         payload_list = self.convert_openapi_to_model(openapi_dict=json_dict)
         # handle duplicate key
         new_payload_list = self.__handle_duplicate_api_label(payload_list)
         return new_payload_list
 
+    def get_api_list_for_openapi(self, json_dict: dict) -> list[dict]:
+        payload_list = self.convert_openapi_to_list(openapi_dict=json_dict)
+        # handle duplicate key
+        new_payload_list = self.__handle_duplicate_api_label_for_dict(payload_list)
+        return new_payload_list
+
     def convert_curl_data_to_list(
         self, curl_file_path: str, url_filter=None
     ) -> list[dict]:
@@ -54,24 +72,26 @@ class ApiModelParser:
         with open(curl_file_path, "rt") as f:
             lines = f.readlines()
             line_num_array = curl_parser.get_curl_line_num_scope(lines=lines)
-            for r in line_num_array:
-                res_dict = curl_parser.split_curl_to_struct(
-                    lines, r[0], r[1], url_filter
-                )
-                req_list = res_dict
-                url_content = curl_parser.parse_url(req_list["original_url"])
-                req_list["temp_api_label"] = (
-                    self.__replace_api_label_chars(url_content["path"][1:])
-                    + "_"
-                    + req_list["method"]
-                )
-                # req_list["header"] = json.dumps(req_list["header"])
-                if url_content["query_params"]:
-                    req_list["query_param"] = json.dumps(url_content["query_params"])
-                else:
-                    req_list["query_param"] = {}
-                req_list["path"] = url_content["path"][1:]
-                payload_list.append(req_list)
+            for s, e in line_num_array:
+                res = curl_parser.split_curl_to_struct(lines, s, e, url_filter)
+                url_content = curl_parser.parse_url(res.get("original_url", ""))
+                path_str = url_content.get("path", "")[1:]
+                method = (res.get("method", "") or "").lower()
+                label = self.__replace_api_label_chars(path_str) + "_" + method
+                req_data = {
+                    "path": path_str,
+                    "header": json.dumps(res.get("header", {})),
+                    "body": res.get("body", ""),
+                    "query_param": (
+                        json.dumps(url_content.get("query_params", {}))
+                        if url_content.get("query_params")
+                        else {}
+                    ),
+                    "original_url": res.get("original_url", ""),
+                    "method": method,
+                    "temp_api_label": label,
+                }
+                payload_list.append(req_data)
         return payload_list
 
     def convert_curl_data_to_model(
@@ -102,6 +122,31 @@ class ApiModelParser:
                 payload_list.append(req_model)
         return payload_list
 
+    def convert_postman_to_list(self, postman_dict: dict) -> list[dict]:
+        api_infos = parse_postman(postman_dict) or []
+        payload_list = []
+        for api_info in api_infos:
+            path = api_info.get("path", "")
+            method = (api_info.get("method", "") or "").lower()
+            label = (
+                self.__handle_http_path(path.split("/"), "_")
+                .replace("{", "")
+                .replace("}", "")
+                + "_"
+                + method
+            )
+            req_data = {
+                "path": path,
+                "header": json.dumps(api_info.get("header", {})),
+                "body": api_info.get("body", {}),
+                "query_param": json.dumps(api_info.get("query_param", {})),
+                "original_url": api_info.get("url", ""),
+                "method": method,
+                "temp_api_label": label,
+            }
+            payload_list.append(req_data)
+        return payload_list
+
     def convert_postman_to_model(self, postman_dict: dict) -> list[ReqData]:
         api_infos = parse_postman(postman_dict)
         payload_list = []
@@ -148,29 +193,78 @@ class ApiModelParser:
             payload_list.append(req_data)
         return payload_list
 
+    def convert_swagger_to_list(self, swagger2_dict: dict) -> list[dict]:
+        swagger_parser = Swagger2Parser(swagger2_dict)
+        api_dict = swagger_parser.get_swagger_api_info()
+        if not api_dict:
+            return []
+        payload_list = []
+        for path, path_info in api_dict.items():
+            parts = path.split("/")
+            path_val = self.__handle_http_path(parts[:-1], "/")
+            label_val = (
+                self.__handle_http_path(parts, "_").replace("{", "").replace("}", "")
+            )
+            body_param = path_info.get("body_param", {})
+            body_val = next(iter(body_param.values()), {})
+            req_data = {
+                "path": path_val,
+                "temp_api_label": label_val,
+                "method": parts[-1],
+                "query_param": json.dumps(path_info.get("query_param", {})),
+                "path_param": json.dumps(path_info.get("path_param", {})),
+                "response": json.dumps(path_info.get("response", {})),
+                "body": json.dumps(body_val),
+            }
+            payload_list.append(req_data)
+        return payload_list
+
     def convert_openapi_to_model(self, openapi_dict: dict) -> list[ReqData]:
         if not openapi_dict:
-            print("openapi dict is null")
             return []
         payload_list = []
         parser = OpenApiParser(openapi_dict)
         api_dict = parser.get_open_api_info()
         if not api_dict:
-            print("check your swagger json")
-            return
+            return []
         for path, path_info in api_dict.items():
-            req_data = ReqData()
-            req_data.path = self.__handle_http_path(path.split("/")[:-1], "/")
-            req_data.temp_api_label = (
-                self.__handle_http_path(path.split("/"), "_")
-                .replace("{", "")
-                .replace("}", "")
+            parts = path.split("/")
+            req = ReqData()
+            req.path = self.__handle_http_path(parts[:-1], "/")
+            req.temp_api_label = (
+                self.__handle_http_path(parts, "_").replace("{", "").replace("}", "")
             )
-            req_data.method = path.split("/")[-1]
-            req_data.query_param = json.dumps(path_info["query_param"])
-            req_data.path_param = json.dumps(path_info["path_param"])
-            req_data.response = json.dumps(path_info.get("response", {}))
-            req_data.body = json.dumps(path_info["body_param"])
+            req.method = parts[-1]
+            req.query_param = json.dumps(path_info.get("query_param", {}))
+            req.path_param = json.dumps(path_info.get("path_param", {}))
+            req.response = json.dumps(path_info.get("response", {}))
+            req.body = json.dumps(path_info.get("body_param", {}))
+            payload_list.append(req)
+        return payload_list
+
+    def convert_openapi_to_list(self, openapi_dict: dict) -> list[dict]:
+        if not openapi_dict:
+            return []
+        payload_list = []
+        parser = OpenApiParser(openapi_dict)
+        api_dict = parser.get_open_api_info()
+        if not api_dict:
+            return []
+        for path, path_info in api_dict.items():
+            parts = path.split("/")
+            path_val = self.__handle_http_path(parts[:-1], "/")
+            label_val = (
+                self.__handle_http_path(parts, "_").replace("{", "").replace("}", "")
+            )
+            req_data = {
+                "path": path_val,
+                "temp_api_label": label_val,
+                "method": parts[-1],
+                "query_param": json.dumps(path_info.get("query_param", {})),
+                "path_param": json.dumps(path_info.get("path_param", {})),
+                "response": json.dumps(path_info.get("response", {})),
+                "body": json.dumps(path_info.get("body_param", {})),
+            }
             payload_list.append(req_data)
         return payload_list
 

{http_content_parser-0.0.21 → http_content_parser-0.0.23}/src/http_content_parser/curl_parser.py

@@ -64,7 +64,7 @@ class CurlParser(object):
                 line_num_array.append([start_num, num])
         return line_num_array
 
-    def split_curl_to_struct(self, lines, s, e, url_filter=None) -> dict:
+    def split_curl_to_struct_old(self, lines, s, e, url_filter=None) -> dict:
         req_data = {}
         header = {}
         reduced_lines = self.reduce_curl_data_part(lines=lines, s=s, e=e)
@@ -109,7 +109,6 @@ class CurlParser(object):
                 else:
                     curl_data = line_i_list[0]
                 body = re.sub(r"\n\s*", "", curl_data.replace(" \\\n", ""))
-                # body = re.sub(r"\s*", "", curl_data.replace(" \\\n", ""))
                 req_data["body"] = body[:-1]
 
                 if not req_data.get("method"):
@@ -120,3 +119,82 @@ class CurlParser(object):
             req_data["method"] = "get"
 
         return req_data
+
+    def split_curl_to_struct(self, lines, s, e, url_filter=None) -> dict:
+        req_data = {}
+        header = {}
+
+        reduced_lines = self.reduce_curl_data_part(lines=lines, s=s, e=e)
+
+        for line in reduced_lines:
+            line = str(line).strip()
+
+            # ---------------------------
+            # 解析 method 和 URL
+            # ---------------------------
+            if line.startswith("curl"):
+                parts = line.split()
+
+                for part in parts:
+                    p = part.strip("'\"")
+
+                    # method
+                    if p.lower() in ["get", "post", "put", "delete"]:
+                        req_data["method"] = p.lower()
+
+                    # URL
+                    elif p.startswith("http"):
+                        req_data["original_url"] = p
+                    elif "/" in p and "://" not in p:  # Postman 样式
+                        req_data["original_url"] = p
+
+                # url_filter
+                if url_filter and url_filter not in line:
+                    continue
+
+            # ---------------------------
+            # -X METHOD
+            # ---------------------------
+            elif line.startswith("-X") or " -X " in line:
+                # 例如：-X POST
+                parts = line.replace("\\", "").split()
+                if len(parts) >= 2:
+                    req_data["method"] = parts[1].strip("'\"").lower()
+
+            # ---------------------------
+            # Header 解析
+            # ---------------------------
+            elif line.startswith("-H") or "--header" in line:
+                # 支持各种写法
+                m = re.search(r"'(.+?):\s*(.+?)'", line)
+                if m:
+                    key = m.group(1)
+                    val = m.group(2)
+                    header[key] = val
+
+            # ---------------------------
+            # Body 解析
+            # ---------------------------
+            elif "--data" in line or "--data-raw" in line:
+                # 去掉行尾的转义符
+                clean = line.replace("\\\n", "").strip()
+
+                # 提取单引号内容
+                m = re.search(r"'(.*)'$", clean)
+                if m:
+                    req_data["body"] = m.group(1)
+                else:
+                    # 兼容无引号情况
+                    req_data["body"] = clean.split("--data")[-1].strip()
+
+                # 如果没有 method，则默认 POST
+                req_data.setdefault("method", "post")
+
+        # ---------------------------
+        # 最终补齐数据
+        # ---------------------------
+        req_data.setdefault("method", "get")
+        req_data["header"] = header
+        req_data.setdefault("original_url", "")
+
+        return req_data

http_content_parser-0.0.23/src/http_content_parser/postman_parser.py

@@ -0,0 +1,123 @@
+# -*- coding: UTF-8 -*-
+import json
+import re
+
+
+def parse_postman(json_file: dict) -> list:
+    items = json_file.get("item")
+    result = []
+
+    def _collect_vars(obj: dict) -> dict:
+        vars_list = obj.get("variable", [])
+        m = {}
+        for it in vars_list:
+            k = it.get("key") or it.get("name")
+            v = it.get("value")
+            if k:
+                m[str(k)] = v if v is not None else ""
+        return m
+
+    root_vars = _collect_vars(json_file)
+    pattern = re.compile(r"\{\{([^{}]+)\}\}")
+
+    def _resolve(s: str, vars_map: dict) -> str:
+        if not isinstance(s, str) or not s:
+            return s
+
+        def repl(m):
+            key = m.group(1).strip()
+            return str(vars_map.get(key, m.group(0)))
+
+        return pattern.sub(repl, s)
+
+    def _walk(item_obj: dict, inherited_vars: dict):
+        local_vars = _collect_vars(item_obj)
+        vars_map = {**inherited_vars, **local_vars}
+        child_items = item_obj.get("item")
+        if child_items:
+            for child in child_items:
+                _walk(child, vars_map)
+            return
+        if not item_obj.get("request"):
+            return
+        postman_request = item_obj["request"]
+        api_info = {}
+        api_info["method"] = postman_request.get("method", "")
+        headers = postman_request.get("header", [])
+        api_info["header"] = {
+            hs.get("key"): _resolve((hs.get("value") or ""), vars_map)
+            for hs in headers
+            if hs.get("key")
+        }
+        url_obj = postman_request.get("url", {})
+        api_info["url"] = _resolve(url_obj.get("raw", ""), vars_map)
+        path_list = url_obj.get("path")
+        if not path_list:
+            return
+        api_info["path"] = "/".join([_resolve(p, vars_map) for p in path_list if p])
+        api_info["query_param"] = {
+            qs.get("key"): _resolve((qs.get("value") or ""), vars_map)
+            for qs in url_obj.get("query", [])
+            if qs.get("key")
+        }
+        body_obj = postman_request.get("body")
+        body_val = {}
+        if body_obj and body_obj.get("mode") in body_obj:
+            temp = body_obj[body_obj["mode"]]
+            if isinstance(temp, str):
+                temp = _resolve(temp.replace("\\n", "").replace("\n", ""), vars_map)
+                if temp:
+                    try:
+                        temp = json.loads(temp)
+                    except Exception:
+                        pass
+            body_val = temp
+        api_info["body"] = body_val
+        result.append(api_info)
+
+    if items:
+        for it in items:
+            _walk(it, root_vars)
+    return result
+
+def parse_postman_bak(json_file: dict) -> list:
+    # TODO 如果postman中有变量,则无法解析变量的值.
+    api_info_list = []
+    items = json_file.get("item")
+    if items:
+        for item in items:
+            if item.get("item"):
+                api_info_list.extend(parse_postman(item))
+            else:
+                api_info = {}
+                if not item.get("request"):
+                    continue
+                postman_request = item["request"]
+                api_info["method"] = postman_request["method"]
+                api_info["header"] = {}
+                for hs in postman_request["header"]:
+                    api_info["header"][hs["key"]] = hs["value"]
+                api_info["url"] = postman_request["url"]["raw"]
+                if not postman_request["url"].get("path"):
+                    continue
+                api_info["path"] = "/".join(postman_request["url"]["path"])
+                api_info["query_param"] = {}
+                if postman_request["url"].get("query"):
+                    for qs in postman_request["url"]["query"]:
+                        api_info["query_param"][qs["key"]] = qs["value"]
+                api_info["body"] = {}
+                if postman_request.get("body"):
+                    temp = postman_request["body"][postman_request["body"]["mode"]]
+                    if isinstance(temp, str):
+                        # \\n 只会出现在value中,一般是用户自己定义的
+                        temp = temp.replace("\\n", "").replace("\n", "")
+                        if temp:
+                            # parse json string
+                            try:
+                                temp = json.loads(temp)
+                            except json.JSONDecodeError:
+                                # Not a valid JSON, treat as a plain string
+                                pass
+                    api_info["body"] = temp
+                api_info_list.append(api_info)
+    return api_info_list

{http_content_parser-0.0.21 → http_content_parser-0.0.23}/tests/test_api_model_parser.py

@@ -1,3 +1,4 @@
+import json
 from http_content_parser.api_parser import ApiModelParser
 
 
@@ -7,4 +8,4 @@ api_parser = ApiModelParser()
 def test_curl_parser():
     curl_file = "./tmp"
     api_info = api_parser.get_api_list_for_curl(curl_file=curl_file)
-    print(api_info)
+    print(json.dumps(api_info, indent=4))

http_content_parser-0.0.23/tests/test_postman.py

@@ -0,0 +1,61 @@
+import json
+
+from http_content_parser.postman_parser import parse_postman
+
+
+def test_parse_postman_basic_and_vars():
+    collection = {
+        "variable": [
+            {"key": "host", "value": "api.example.com"},
+            {"name": "token", "value": "abc123"},
+        ],
+        "item": [
+            {
+                "name": "Folder",
+                "item": [
+                    {
+                        "name": "GetUser",
+                        "request": {
+                            "method": "GET",
+                            "header": [
+                                {"key": "Authorization", "value": "Bearer {{token}}"}
+                            ],
+                            "url": {
+                                "raw": "https://{{host}}/v1/users?id={{uid}}",
+                                "path": ["v1", "users"],
+                                "query": [{"key": "id", "value": "{{uid}}"}],
+                            },
+                        },
+                        "variable": [{"key": "uid", "value": "42"}],
+                    }
+                ],
+            }
+        ],
+    }
+    res = parse_postman(collection)
+    assert isinstance(res, list)
+    assert len(res) == 1
+    item = res[0]
+    assert item["method"] == "GET"
+    assert item["path"] == "v1/users"
+    assert item["url"].startswith("https://api.example.com")
+    assert item["query_param"]["id"] == "42"
+    assert item["header"]["Authorization"] == "Bearer abc123"
+
+
+def test_parse_postman_no_global_accumulate():
+    c = {
+        "item": [
+            {
+                "request": {
+                    "method": "GET",
+                    "header": [],
+                    "url": {"raw": "https://x/a", "path": ["a"], "query": []},
+                }
+            }
+        ]
+    }
+    r1 = parse_postman(c)
+    r2 = parse_postman(c)
+    assert len(r1) == 1
+    assert len(r2) == 1

http_content_parser-0.0.21/src/http_content_parser/postman_parser.py

@@ -1,44 +0,0 @@
-# -*- coding: UTF-8 -*-
-import json
-
-
-api_info_list = []
-
-
-def parse_postman(json_file: dict) -> list:
-    # TODO 如果postman中有变量,则无法解析变量的值.
-    items = json_file.get("item")
-    if items:
-        for item in items:
-            if item.get("item"):
-                parse_postman(item)
-            else:
-                api_info = {}
-                if not item.get("request"):
-                    continue
-                postman_request = item["request"]
-                api_info["method"] = postman_request["method"]
-                api_info["header"] = {}
-                for hs in postman_request["header"]:
-                    api_info["header"][hs["key"]] = hs["value"]
-                api_info["url"] = postman_request["url"]["raw"]
-                if not postman_request["url"].get("path"):
-                    continue
-                api_info["path"] = "/".join(postman_request["url"]["path"])
-                api_info["query_param"] = {}
-                if postman_request["url"].get("query"):
-                    for qs in postman_request["url"]["query"]:
-                        api_info["query_param"][qs["key"]] = qs["value"]
-                api_info["body"] = {}
-                if postman_request.get("body"):
-                    temp = postman_request["body"][postman_request["body"]["mode"]]
-                    if isinstance(temp, str):
-                        # \\n 只会出现在value中,一般是用户自己定义的
-                        temp = temp.replace("\\n", "").replace("\n", "")
-                        if temp:
-                            # beautify json string
-                            temp = json.dumps(temp) # 转换为 JSON 格式的字符串
-                            temp = json.loads(temp)
-                    api_info["body"] = temp
-                api_info_list.append(api_info)
-    return api_info_list