py2ls 0.2.4.1__py3-none-any.whl → 0.2.4.3__py3-none-any.whl

py2ls/fetch_update.py

@@ -13,45 +13,63 @@ def usage_pd(
         sp = get_soup(url, driver="se")
         return fetch(sp, where="dt")[0]
 
+
     if dir_save is None:
         if "mac" in get_os():
             dir_save = "/Users/macjianfeng/Dropbox/github/python/py2ls/py2ls/data/"
         else:
             dir_save = "Z:\\Jianfeng\\temp\\"
     sp = get_soup(url, driver="se")
-    links_all = fetch(sp, where="a", get="href", class_="reference internal")
-    filtered_links = unique(
-        [i for i in links_all if any([i.startswith(cond) for cond in ["pandas"]])]
-    )
-    links = [
-        "https://pandas.pydata.org/docs/reference/api/" + i for i in filtered_links
+    links_all4lev1 = fetch(sp, where="a", get="href", class_="reference internal")
+    links_level_1 = [
+        strcmp(link, links_all4lev1)[0].replace(
+            "../", "https://pandas.pydata.org/docs/reference/"
+        )
+        for link in links_all4lev1
+        if not link.startswith("pandas")
     ]
-
-    usages = [get_usage(i) for i in links]
     dict_usage = {}
-    for usage, link in zip(usages, links):
-        if usage.startswith("DataFrame"):
-            usage = usage.replace("DataFrame", "df")
-        if usage.startswith("pandas"):
-            usage = usage.replace("pandas", "pd")
-        if usage.endswith("[source]#"):
-            usage = usage.replace("[source]#", "")
-        if usage.endswith("#"):
-            usage = usage.replace("#", "")
-        str2rm = ["class", "property"]
-        for str2rm_ in str2rm:
-            if usage.startswith(str2rm_):
-                usage = usage.replace(str2rm_, "")
-        funcname = ssplit(usage, by="(")[0]
-        dict_usage.update({funcname: usage + f"\n{link}"})
+    for link_level_1 in links_level_1:
+        sp = get_soup(link_level_1, driver="se")
+        links_all = fetch(sp, where="a", get="href", class_="reference internal")
+
+        filtered_links = unique(
+            [
+                i
+                for i in links_all
+                if any([i.startswith(cond) for cond in ["pandas", "api"]])
+            ]
+        )
+        links = [
+            (
+                "https://pandas.pydata.org/docs/reference/api/" + i
+                if not i.startswith("api")
+                else "https://pandas.pydata.org/docs/reference/" + i
+            )
+            for i in filtered_links
+        ]
+        usages = [get_usage(i) for i in links]
+        for usage, link in zip(usages, links):
+            if usage.startswith("DataFrame"):
+                usage = usage.replace("DataFrame", "df")
+            if usage.startswith("pandas"):
+                usage = usage.replace("pandas", "pd")
+            if usage.endswith("[source]#"):
+                usage = usage.replace("[source]#", "")
+            if usage.endswith("#"):
+                usage = usage.replace("#", "")
+            str2rm = ["class", "property"]
+            for str2rm_ in str2rm:
+                if usage.startswith(str2rm_):
+                    usage = usage.replace(str2rm_, "")
+            funcname = ssplit(usage, by="(")[0]
+            dict_usage.update({funcname: usage + f"\n{link}"})
     # save to local
     dir_save += "/" if not dir_save.endswith("/") else ""
     fsave(
         dir_save + "usages_pd.json",
         dict_usage,
     )
-
-
 def usage_sns(
     url="https://seaborn.pydata.org/generated/seaborn.swarmplot.html",
     dir_save=None,
@@ -119,8 +137,8 @@ def usage_sns(
 
 def main():
     # update pandas usage to local
-    update_pd_usages()
-    # update_sns_usages()
+    usage_pd()
+    # usage_sns()
 
 
 if __name__ == "__main__":