py2ls 0.2.4.25__py3-none-any.whl → 0.2.4.26__py3-none-any.whl

py2ls/netfinder.py

@@ -626,7 +626,7 @@ def filter_links(links, contains="html", driver="requ", booster=False):
         )
         if condition:
             filtered_links.append(link)
-    return filtered_links
+    return ips.unique(filtered_links)
 
 
 def find_domain(links):
@@ -717,7 +717,7 @@ def downloader(
     kind=[".pdf"],
     contains=None,
     rm_folder=False,
-    booster=False,
+    booster=True,# use find_links
     verbose=True,
     timeout=30,
     n_try=3,
@@ -726,7 +726,7 @@ def downloader(
 
     from requests.exceptions import ChunkedEncodingError, ConnectionError
 
-    if verbose:
+    if verbose and ips.run_once_within():
         print(
             "usage: downloader(url, dir_save=None, kind=['.pdf','xls'], contains=None, booster=False)"
         )
@@ -734,8 +734,11 @@ def downloader(
     def fname_corrector(fname, ext):
         if not ext.startswith("."):
             ext = "." + ext
-        if not fname.endswith("ext"):  # if not ext in fname:
+        if not fname.endswith(ext):  # if not ext in fname:
             fname = fname[: -len(ext)] + ext
+        if not any(fname[: -len(ext)]):
+            from datetime import datetime
+            fname = datetime.now().strftime("%H%M%S") + ext
         return fname
 
     def check_and_modify_filename(directory, filename):
@@ -784,8 +787,8 @@ def downloader(
             kind[i] = "." + kind[i]
     file_links_all = []
     for kind_ in kind:
-        if isinstance(contains, str):
-            contains = [contains]
+        # if isinstance(contains, str):
+        #     contains = [contains]
         if isinstance(url, str):
             if any(ext in url for ext in kind):
                 file_links = [url]
@@ -799,7 +802,7 @@ def downloader(
                 if contains is not None:
                     file_links = filter_links(links_all, contains=contains + kind_)
                 else:
-                    file_links = links_all  # filter_links(links_all, contains=kind_)
+                    file_links = filter_links(links_all, contains=kind_)#links_all  # 
         elif isinstance(url, list):
             links_all = url
             if contains is not None:
@@ -812,6 +815,7 @@ def downloader(
                 file_links = filter_links(links_all, contains=contains + kind_)
             else:
                 file_links = filter_links(links_all, contains=kind_)
+        file_links=ips.unique(file_links)
         if verbose:
             if file_links:
                 from pprint import pp
@@ -825,6 +829,7 @@ def downloader(
             file_links_all = [file_links]
         elif isinstance(file_links, list):
             file_links_all.extend(file_links)
+    file_links_all=ips.unique(file_links_all)
     if dir_save:
         if rm_folder:
             ips.rm_folder(dir_save)
@@ -847,7 +852,7 @@ def downloader(
                         )
                         if ext is None:
                             ext = kind_
-                        print("ehereerere", ext)
+
                         if ext:
                             corrected_fname = fname_corrector(fnames[idx], ext)
                             corrected_fname = check_and_modify_filename(
@@ -860,13 +865,13 @@ def downloader(
                                     datetime.now().strftime("%y%m%d_%H%M%S_")
                                     + corrected_fname
                                 )
-                            fpath_tmp = os.path.join(dir_save, corrected_fname)
+                            fpath_tmp = os.path.join(dir_save, corrected_fname) 
                             with open(fpath_tmp, "wb") as file:
                                 for chunk in response.iter_content(chunk_size=8192):
                                     if chunk:  # Filter out keep-alive chunks
                                         file.write(chunk)
                             if verbose:
-                                print(f"Done! {fnames[idx]}")
+                                print(f"Done⤵{fnames[idx]}")
                         else:
                             if verbose:
                                 print(f"Unknown file type for {file_link}")
@@ -886,16 +891,7 @@ def downloader(
 
             if itry == n_try:
                 print(f"Failed to download {file_link} after {n_try} attempts.")
-
-        # print(f"\n{len(fnames)} files were downloaded:")
-        if verbose:
-            from pprint import pp
-
-            if corrected_fname:
-                pp(corrected_fname)
-                print(f"\n\nsaved @:\n{dir_save}")
-            else:
-                pp(fnames)
+ 
 
 
 def find_img(url, driver="request", dir_save="images", rm_folder=False, verbose=True):