http-content-parser 0.0.17__tar.gz → 0.0.19__tar.gz

{http_content_parser-0.0.17 → http_content_parser-0.0.19}/.gitignore

@@ -158,4 +158,7 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
-.vscode/
+.vscode/
+
+# 
+tmp

{http_content_parser-0.0.17 → http_content_parser-0.0.19}/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: http_content_parser
-Version: 0.0.17
+Version: 0.0.19
 Summary: parse http's payload and response
-Author-email: max su <suleiabc@gmail.com>
+Author-email: leo <suleiabc@gmail.com>
 License-File: LICENSE
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -24,5 +24,5 @@ python3 -m build
 
 ```bash
 rm -f dist/*
- python3 -m twine upload dist/*
+python3 -m twine upload dist/*
 ```

{http_content_parser-0.0.17 → http_content_parser-0.0.19}/README.md

@@ -12,5 +12,5 @@ python3 -m build
 
 ```bash
 rm -f dist/*
- python3 -m twine upload dist/*
+python3 -m twine upload dist/*
 ```

{http_content_parser-0.0.17 → http_content_parser-0.0.19}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
 
 [project]
 name = "http_content_parser"
-version = "0.0.17"
-authors = [{ name = "max su", email = "suleiabc@gmail.com" }]
+version = "0.0.19"
+authors = [{ name = "leo", email = "suleiabc@gmail.com" }]
 description = "parse http's payload and response"
 readme = "README.md"
 requires-python = ">=3.8"

http_content_parser-0.0.19/src/http_content_parser/api_parser.py

@@ -0,0 +1,165 @@
+# -*- coding: UTF-8 -*-
+import copy
+import json
+import re
+from http_content_parser.curl_parser import CurlParser
+from http_content_parser.openapi_parser import OpenApiParser
+from http_content_parser.postman_parser import parse_postman
+from http_content_parser.req_data import ReqData
+from http_content_parser.swagger2_parser import Swagger2Parser
+
+
+class ApiModelParser:
+    def get_api_model_for_curl(self, curl_file, curl_filter=None) -> list[ReqData]:
+        # convert curl
+        payload_list = self.convert_curl_data_to_model(
+            curl_file_path=curl_file, url_filter=curl_filter
+        )
+        # handle duplicate key
+        new_payload_list = self.__handle_duplicate_api_label(payload_list)
+        return new_payload_list
+
+    def get_api_model_for_postman(self, json_dict: dict) -> list[ReqData]:
+        payload_list = self.convert_postman_to_model(postman_dict=json_dict)
+        # handle duplicate key
+        new_payload_list = self.__handle_duplicate_api_label(payload_list)
+        return new_payload_list
+
+    def get_api_model_for_swagger(self, json_dict: dict) -> list[ReqData]:
+        payload_list = self.convert_swagger_to_model(swagger2_dict=json_dict)
+        # handle duplicate key
+        new_payload_list = self.__handle_duplicate_api_label(payload_list)
+        return new_payload_list
+
+    def get_api_model_for_openapi(self, json_dict: dict) -> list[ReqData]:
+        payload_list = self.convert_openapi_to_model(openapi_dict=json_dict)
+        # handle duplicate key
+        new_payload_list = self.__handle_duplicate_api_label(payload_list)
+        return new_payload_list
+
+    def convert_curl_data_to_model(
+        self, curl_file_path: str, url_filter=None
+    ) -> list[ReqData]:
+        curl_parser = CurlParser()
+        payload_list = []
+        with open(curl_file_path, "rt") as f:
+            lines = f.readlines()
+            line_num_array = curl_parser.get_curl_line_num_scope(lines=lines)
+            for r in line_num_array:
+                res_dict = curl_parser.split_curl_to_struct(
+                    lines, r[0], r[1], url_filter
+                )
+                req_model = ReqData(dd=res_dict)
+                url_content = curl_parser.parse_url(req_model.original_url)
+                req_model.temp_api_label = (
+                    self.__replace_api_label_chars(url_content["path"][1:])
+                    + "_"
+                    + req_model.method
+                )
+                req_model.header = json.dumps(req_model.header)
+                req_model.query_param = json.dumps(url_content["query_params"])
+                req_model.path = url_content["path"][1:]
+                payload_list.append(req_model)
+        return payload_list
+
+    def convert_postman_to_model(self, postman_dict: dict) -> list[ReqData]:
+        api_infos = parse_postman(postman_dict)
+        payload_list = []
+        for api_info in api_infos:
+            req_data = ReqData()
+            req_data.path = api_info["path"]
+            req_data.header = json.dumps(api_info["header"])
+            req_data.body = api_info["body"]
+            req_data.query_param = json.dumps(api_info["query_param"])
+            req_data.original_url = api_info["url"]
+            req_data.method = api_info["method"].lower()
+            req_data.temp_api_label = (
+                self.__handle_http_path(api_info["path"].split("/"), "_")
+                .replace("{", "")
+                .replace("}", "")
+                + "_"
+                + api_info["method"].lower()
+            )
+            payload_list.append(req_data)
+        return payload_list
+
+    def convert_swagger_to_model(self, swagger2_dict: dict) -> list[ReqData]:
+        swagger_parser = Swagger2Parser(swagger2_dict)
+        api_dict = swagger_parser.get_swagger_api_info()
+        if not api_dict:
+            print("check your swagger json")
+            return
+        payload_list = []
+        for path, path_info in api_dict.items():
+            req_data = ReqData()
+            req_data.path = self.__handle_http_path(path.split("/")[:-1], "/")
+            req_data.temp_api_label = (
+                self.__handle_http_path(path.split("/"), "_")
+                .replace("{", "")
+                .replace("}", "")
+            )
+            req_data.method = path.split("/")[-1]
+            req_data.query_param = json.dumps(path_info["query_param"])
+            req_data.path_param = json.dumps(path_info["path_param"])
+            req_data.response = json.dumps(path_info.get("response", {}))
+            # swagger中body第一层和第二层key重复,只取第二层后的数据
+            for _, v in path_info["body_param"].items():
+                req_data.body = json.dumps(v)
+            payload_list.append(req_data)
+        return payload_list
+
+    def convert_openapi_to_model(self, openapi_dict: dict) -> list[ReqData]:
+        if not openapi_dict:
+            print("openapi dict is null")
+            return []
+        payload_list = []
+        parser = OpenApiParser(openapi_dict)
+        api_dict = parser.get_open_api_info()
+        if not api_dict:
+            print("check your swagger json")
+            return
+        for path, path_info in api_dict.items():
+            req_data = ReqData()
+            req_data.path = self.__handle_http_path(path.split("/")[:-1], "/")
+            req_data.temp_api_label = (
+                self.__handle_http_path(path.split("/"), "_")
+                .replace("{", "")
+                .replace("}", "")
+            )
+            req_data.method = path.split("/")[-1]
+            req_data.query_param = json.dumps(path_info["query_param"])
+            req_data.path_param = json.dumps(path_info["path_param"])
+            req_data.response = json.dumps(path_info.get("response", {}))
+            req_data.body = json.dumps(path_info["body_param"])
+            payload_list.append(req_data)
+        return payload_list
+
+    def __handle_http_path(self, path_list: list, split_char: str) -> str:
+        if not path_list:
+            return ""
+        url_path = ""
+        for u in path_list:
+            if u:
+                url_path += u + split_char
+        return url_path[:-1]
+
+    def __replace_api_label_chars(self, string: str) -> str:
+        pattern = r"[+/@?=.]"  # 定义要匹配的特殊字符模式
+        replacement = "_"  # 替换为的字符串
+
+        new_string = re.sub(pattern, replacement, string)
+        return new_string
+
+    def __handle_duplicate_api_label(
+        self, payload_list: list[ReqData]
+    ) -> list[ReqData]:
+        key_filter = {}
+        new_payload_list = copy.deepcopy(payload_list)
+        for payload, p_copy in zip(payload_list, new_payload_list):
+            k = payload.temp_api_label
+            if k in key_filter.keys():
+                p_copy.temp_api_label = k + "_" + str(key_filter[k])
+                key_filter[k] += 1
+            else:
+                key_filter[k] = 2
+        return new_payload_list

{http_content_parser-0.0.17 → http_content_parser-0.0.19}/src/http_content_parser/curl_parser.py

@@ -5,6 +5,8 @@ from urllib.parse import urlparse, parse_qs
 
 class CurlParser(object):
     def parse_url(self, url: str) -> dict:
+        if "://" not in url:
+            url = "http://" + url
         # 解析 URL
         parsed_url = urlparse(url)
         # 获取各个组成部分
@@ -62,7 +64,7 @@ class CurlParser(object):
                 line_num_array.append([start_num, num])
         return line_num_array
 
-    def split_curl_to_struct(self, lines, s, e, curl_filter=None) -> dict:
+    def split_curl_to_struct(self, lines, s, e, url_filter=None) -> dict:
         req_data = {}
         header = {}
         reduced_lines = self.reduce_curl_data_part(lines=lines, s=s, e=e)
@@ -70,8 +72,8 @@ class CurlParser(object):
             lines_i_str = str(reduced_lines[i])
             line_i_list = lines_i_str.split(" ")
             if "curl" in lines_i_str:
-                if curl_filter:
-                    if curl_filter not in lines_i_str:
+                if url_filter:
+                    if url_filter not in lines_i_str:
                         # 如果curl_filter 不存在于当前Url中,则跳过本次循环
                         continue
                 for line_sub in line_i_list:
@@ -106,7 +108,8 @@ class CurlParser(object):
                     curl_data = line_i_list[1]
                 else:
                     curl_data = line_i_list[0]
-                body = re.sub( r"\n\s*", "", curl_data.replace(" \\\n", ""))
+                body = re.sub(r"\n\s*", "", curl_data.replace(" \\\n", ""))
+                # body = re.sub(r"\s*", "", curl_data.replace(" \\\n", ""))
                 req_data["body"] = body[:-1]
 
                 if not req_data.get("method"):

{http_content_parser-0.0.17 → http_content_parser-0.0.19}/src/http_content_parser/generate_api_file.py

@@ -11,6 +11,7 @@ from http_content_parser.openapi_parser import OpenApiParser
 from http_content_parser.postman_parser import parse_postman
 
 
+# TODO 不直接生成yaml文件,只生成dict,方便后续扩展
 class GenerateApiFile:
     def __init__(self) -> None:
         pass
@@ -39,7 +40,7 @@ class GenerateApiFile:
     def produce_api_yaml_for_curl(self, curl_file, yaml_file, curl_filter=None):
         # convert curl
         payload_list = self.convert_curl_data_to_model(
-            curl_file_path=curl_file, curl_filter=curl_filter
+            curl_file_path=curl_file, url_filter=curl_filter
         )
         # handle duplicate key
         new_payload_list = self.handle_duplicate_yaml_key(payload_list)
@@ -56,11 +57,11 @@ class GenerateApiFile:
                 p_copy.temp_api_label = k + "_" + str(key_filter[k])
                 key_filter[k] += 1
             else:
-                key_filter[k] = 1
+                key_filter[k] = 2
         return new_payload_list
 
     def convert_curl_data_to_model(
-        self, curl_file_path, curl_filter=None
+        self, curl_file_path, url_filter=None
     ) -> list[ReqData]:
         curl_parser = CurlParser()
         payload_list = []
@@ -69,7 +70,7 @@ class GenerateApiFile:
             line_num_array = curl_parser.get_curl_line_num_scope(lines=lines)
             for r in line_num_array:
                 res_dict = curl_parser.split_curl_to_struct(
-                    lines, r[0], r[1], curl_filter
+                    lines, r[0], r[1], url_filter
                 )
                 req_model = ReqData(dd=res_dict)
                 url_content = curl_parser.parse_url(req_model.original_url)
@@ -79,36 +80,14 @@ class GenerateApiFile:
                     + req_model.method
                 )
                 req_model.header = json.dumps(req_model.header)
-                req_model.query_param = json.dumps(url_content["query_params"])
+                if url_content["query_params"]:
+                    req_model.query_param = json.dumps(url_content["query_params"])
+                else:
+                    req_model.query_param = {}
                 req_model.path = url_content["path"][1:]
                 payload_list.append(req_model)
         return payload_list
 
-    def convert_curl_data_to_model_old(
-        self, curl_file_path, curl_filter=None
-    ) -> list[ReqData]:
-        curl_parser = CurlParser()
-        payload_list = []
-        with open(curl_file_path, "rt") as f:
-            lines = f.readlines()
-            line_num_array = curl_parser.get_curl_line_num_scope(lines=lines)
-            for r in line_num_array:
-                res_dict = curl_parser.split_curl_to_struct(
-                    lines, r[0], r[1], curl_filter
-                )
-                template = ReqData(dd=res_dict)
-                split_url = curl_parser.split_url(template.original_url, "_")
-                template.temp_api_label = (
-                    self.replace_api_label_chars(split_url["url_path"])
-                    + template.method
-                )
-                template.header = json.dumps(template.header)
-                template.query_param = json.dumps(split_url["url_params"])
-                split_url_origin = curl_parser.split_url(template.original_url, "/")
-                template.path = split_url_origin["url_path"][:-1]
-                payload_list.append(template)
-        return payload_list
-
     def produce_api_yaml_for_swagger2(self, swagger2_dict, yaml_file):
         swagger_parser = Swagger2Parser(swagger2_dict)
         api_dict = swagger_parser.get_swagger_api_info()

{http_content_parser-0.0.17 → http_content_parser-0.0.19}/src/http_content_parser/req_data.py

@@ -1,7 +1,7 @@
 # -*- coding: UTF-8 -*-
 
 """
-__author__ = maxsulei
+__author__ = leo
 """
 
 

http_content_parser-0.0.19/tests/test_api_model_parser.py

@@ -0,0 +1,10 @@
+from http_content_parser.api_parser import ApiModelParser
+
+
+api_parser = ApiModelParser()
+
+
+def test_curl_parser():
+    curl_file = "./tmp"
+    api_info = api_parser.get_api_model_for_curl(curl_file=curl_file)
+    print(api_info)

http_content_parser-0.0.19/tests/test_curl.py

@@ -0,0 +1,20 @@
+import json
+
+from http_content_parser.generate_api_file import GenerateApiFile
+
+
+def test_curl():
+    gaf = GenerateApiFile()
+    # with open("./postman.json", "r") as f:
+    #     json_dict = json.load(f)
+    # gaf.produce_api_yaml_for_postman(json_dict, "./test.yaml")
+    curl_file = (
+        "./tmp"
+    )
+    gaf.produce_api_yaml_for_curl(curl_file=curl_file, yaml_file="api.yaml")
+
+
+def test_for():
+    a = "#!/usr/bin/env bash  \t\n \n echo \\n 66 >> /root/pre23"
+    b = a.replace("\n", "")
+    print(b)

http_content_parser-0.0.17/tests/test_curl.py

@@ -1,20 +0,0 @@
-import json
-
-from http_content_parser.generate_api_file import GenerateApiFile
-
-
-def test_curl():
-    gaf = GenerateApiFile()
-    # with open("./postman.json", "r") as f:
-    #     json_dict = json.load(f)
-    # gaf.produce_api_yaml_for_postman(json_dict, "./test.yaml")
-    curl_file = ""
-    res = gaf.produce_api_yaml_for_curl(curl_file=curl_file, yaml_file="api.yaml")
-
-
-def test_for():
-    data = {"name": "John", "age": 30, "city": "New York"}
-    temp = '{    "job_id":{{job_id}},    "product_id": 4,    "local_user_id": 0}'
-    temp = json.dumps(data)
-    temp = json.loads(temp)
-    print(temp)