oafuncs 0.0.98.46__py3-none-any.whl → 0.0.98.47__py3-none-any.whl

oafuncs/oa_down/literature.py

@@ -12,7 +12,7 @@ from oafuncs.oa_down.user_agent import get_ua
 from oafuncs.oa_file import remove
 from oafuncs.oa_data import ensure_list
 
-__all__ = ["download5doi", "download5doi_via_unpaywall"]
+__all__ = ["download5doi"]
 
 
 def _get_file_size(file_path, unit="KB"):
@@ -368,129 +368,6 @@ def download5doi(
         dl.download_pdf()
 
 
-# ------------------------------- 合规替代方案（Open Access 优先） -------------------------------
-def _get_oa_pdf_url_from_unpaywall(doi: str, email: str | None) -> str | None:
-    """
-    通过 Unpaywall 获取可开放访问的 PDF 链接（若存在）。
-    需要提供 email（Unpaywall 要求标识邮件）。
-    返回 PDF URL 或 None。
-    """
-    if not email:
-        print("[bold yellow]Unpaywall 需要 email 参数；请提供 email 以查询 OA 链接。")
-        return None
-    api = f"https://api.unpaywall.org/v2/{doi}?email={email}"
-    try:
-        r = requests.get(api, timeout=15)
-        if r.status_code != 200:
-            print(f"[bold yellow]Unpaywall 查询失败: HTTP {r.status_code}")
-            return None
-        data = r.json()
-        loc = data.get("best_oa_location") or {}
-        url_for_pdf = loc.get("url_for_pdf") or loc.get("url")
-        if url_for_pdf and url_for_pdf.lower().endswith(".pdf"):
-            return url_for_pdf
-        # 有些 OA 链接是落在 landing page，再尝试从记录的所有位置挑选 pdf
-        for k in ("oa_locations", "oa_location"):
-            entries = data.get(k) or []
-            if isinstance(entries, dict):
-                entries = [entries]
-            for e in entries:
-                u = e.get("url_for_pdf") or e.get("url")
-                if u and ".pdf" in u.lower():
-                    return u
-    except Exception as e:
-        print(f"[bold yellow]Unpaywall 查询异常: {e}")
-    return None
-
-
-def _download_pdf_from_url(url: str, dest_path: Path, headers: dict | None = None) -> bool:
-    """
-    给定合法的 PDF 下载 URL，下载保存到 dest_path。
-    返回 True/False 表示是否成功。
-    """
-    headers = headers or {"User-Agent": str(get_ua()), "Accept": "application/pdf"}
-    try:
-        with requests.get(url, headers=headers, stream=True, timeout=30) as r:
-            if r.status_code != 200 or "application/pdf" not in r.headers.get("Content-Type", "").lower():
-                # 仍可能是 PDF（某些服务器未正确设置头），尝试保存但标注提示
-                if r.status_code != 200:
-                    print(f"[bold yellow]下载失败: HTTP {r.status_code}")
-                    return False
-            dest_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(dest_path, "wb") as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    if chunk:
-                        f.write(chunk)
-        return True
-    except Exception as e:
-        print(f"[bold yellow]下载异常: {e}")
-        return False
-
-
-def download5doi_via_unpaywall(
-    store_path=None,
-    doi_list=None,
-    txt_file=None,
-    excel_file=None,
-    col_name=r"DOI",
-    email: str | None = None,
-):
-    """
-    优先使用 Unpaywall 获取开放访问（OA）的 PDF 并下载，避免非合规站点。
-
-    参数：
-        store_path: 保存目录
-        doi_list/txt_file/excel_file/col_name: 同 download5doi
-        email: 用于访问 Unpaywall API 的邮箱（必填，否则无法查询）
-    """
-    if not store_path:
-        store_path = Path.cwd()
-    else:
-        store_path = Path(str(store_path))
-    store_path.mkdir(parents=True, exist_ok=True)
-
-    if doi_list:
-        doi_list = ensure_list(doi_list)
-    if txt_file:
-        doi_list = _read_txt(txt_file)
-    if excel_file:
-        doi_list = _read_excel(excel_file, col_name)
-
-    if not doi_list:
-        print("[bold yellow]未提供 DOI 列表。")
-        return
-
-    print(f"[bold cyan]通过 Unpaywall 尝试下载 {len(doi_list)} 篇 OA PDF...")
-    ok, miss = 0, 0
-    for doi in track(doi_list, description="OA downloading..."):
-        # 规范化文件名
-        fname = re.sub(r'[/<>:"?*|]', "_", str(doi)) + ".pdf"
-        dest = store_path / fname
-        if dest.exists() and _get_file_size(dest, unit="KB") > 10:
-            ok += 1
-            continue
-
-        pdf_url = _get_oa_pdf_url_from_unpaywall(str(doi), email=email)
-        if not pdf_url:
-            miss += 1
-            print(f"[bold yellow]未找到 OA PDF: {doi}")
-            continue
-
-        if _download_pdf_from_url(pdf_url, dest):
-            size_kb = _get_file_size(dest, unit="KB")
-            if isinstance(size_kb, (int, float)) and size_kb < 10:
-                dest.unlink(missing_ok=True)
-                miss += 1
-                print(f"[bold yellow]文件过小，疑似异常，已删除: {dest}")
-            else:
-                ok += 1
-                print(f"[bold green]已下载: {dest}")
-        else:
-            miss += 1
-
-    print(f"[bold]完成。成功 {ok} 篇，未获取 {miss} 篇（可能无 OA 版本或需机构访问）。")
-
-
 if __name__ == "__main__":
     store_path = r"F:\AAA-Delete\DOI_Reference\5\pdf"
     excel_file = r"F:\AAA-Delete\DOI_Reference\5\savedrecs.xls"

oafuncs/oa_linux.py

@@ -68,11 +68,18 @@ def running_jobs():
     ids = [job.split()[0] for job in Jobs if job != '']
     return ids
 
-def submit_job(working_dir, script_tmp='run.slurm', script_run='run.slurm', need_node=1, queue_tmp='<queue_name>', queue_list=['dcu', 'bigmem', 'cpu_parallel', 'cpu_single'], max_job=38):
+def submit_job(working_dir=None, script_tmp='run.slurm', script_run='run.slurm', need_node=1, queue_tmp='<queue_name>', queue_list=['dcu', 'bigmem', 'cpu_parallel', 'cpu_single'], max_job=38, wait=False): 
+    '''提交任务到集群，并返回任务ID'''
     from .oa_file import replace_content
     import datetime
+    if working_dir is None:
+        working_dir = os.getcwd()
     os.chdir(working_dir)
     print(f'切换工作目录到: {working_dir}')
+    
+    if need_node > 1 and 'cpu_single' in queue_list:
+        queue_list.remove('cpu_single')
+    
     while True:
         running_job = running_jobs()
         if not running_job or len(running_job) < max_job:
@@ -94,6 +101,7 @@ def submit_job(working_dir, script_tmp='run.slurm', script_run='run.slurm', need
                         time.sleep(30)
                     else:
                         print(f'提交任务成功，{content_sub.strip()}')
+                        job_id = content_sub.strip().split()[-1]
                         break
             else:
                 print('没有足够的计算资源，等待30秒后重试！')
@@ -103,6 +111,20 @@ def submit_job(working_dir, script_tmp='run.slurm', script_run='run.slurm', need
             time.sleep(60)
     print(f'等待10秒后，继续检查任务状态！')
     time.sleep(10)
+    
+    if wait:
+        while True:
+            if job_id in running_jobs():
+                print(f'Time: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
+                print(f'任务{job_id}正在任务队列中...')
+                time.sleep(60)
+            else:
+                print(f'任务{job_id}已完成！')
+                break
+    else:
+        print(f'任务{job_id}已提交，不等待其完成，继续执行后续操作！')
+    
+    return job_id
 
 if __name__ == "__main__":
     pass

{oafuncs-0.0.98.46.dist-info → oafuncs-0.0.98.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.46
+Version: 0.0.98.47
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.46.dist-info → oafuncs-0.0.98.47.dist-info}/RECORD

@@ -6,7 +6,7 @@ oafuncs/oa_draw.py,sha256=zal0Y3RPpN0TCGN4Gw9qLtjQdT6V0ZqpSUBFVOPL0x4,13952
 oafuncs/oa_file.py,sha256=j9NOjxPOeAJsD5Zk4ODmFdVSSgr1CHVPvM1IHXy9RQA,17546
 oafuncs/oa_geo.py,sha256=UbzvUqgT2QP_9B7XSJRL1HDmGu0HnLC5nSP6ZrA5WH4,7177
 oafuncs/oa_help.py,sha256=0J5VaZX-cB0c090KxgmktQJBc0o00FsY-4wB8l5y00k,4178
-oafuncs/oa_linux.py,sha256=eijpxTopzL3GpE5AIzis9vdrbm-A7QBeQesA-divBjE,4627
+oafuncs/oa_linux.py,sha256=reQYcjMff6mHuM5RXzoxM-4lgIYUcOoHp0u2fLY0cnU,5431
 oafuncs/oa_nc.py,sha256=j501NlTuvrDIwNLXbMfE7nPPXdbbL7u9PGDj2l5AtnI,16277
 oafuncs/oa_python.py,sha256=xYMQnM0cGq9xUCtcoMpnN0LG5Rc_s94tai5nC6CNJ3E,4831
 oafuncs/oa_tool.py,sha256=VHx15VqpbzNlVXh0-3nJqcDgLVaECMD1FvxJ_CrV39E,8046
@@ -26,7 +26,7 @@ oafuncs/oa_down/User_Agent-list.txt,sha256=pHaMlElMvZ8TG4vf4BqkZYKqe0JIGkr4kCN0l
 oafuncs/oa_down/__init__.py,sha256=IT6oTqaxuV_mC6AwBut0HtkmnVtEu1MyX0x0oS7TKoA,218
 oafuncs/oa_down/hycom_3hourly.py,sha256=dFXSC_5o-Dic616KrLXir4tEHvCiZt8vGKPEYpXFMmA,57356
 oafuncs/oa_down/idm.py,sha256=vAhRjt_Sb-rKhzFShmSf29QcFTqsHpHXCvTSD1uSXyQ,1455
-oafuncs/oa_down/literature.py,sha256=umz8bqYoVJiFkFviK970iOL7sfwbVWuqHPgRs3a199I,19806
+oafuncs/oa_down/literature.py,sha256=bqRSwYjsPO28WIohGIwPcj0xo9vAz5yhc8Ojx8ybh08,14924
 oafuncs/oa_down/read_proxy.py,sha256=HQpr-Mwn0Z8ICAuf63NUUY9p05E_uNWyWmOK46-73Ec,2866
 oafuncs/oa_down/test_ua.py,sha256=l8MCD6yU2W75zRPTDKUZTJhCWNF9lfk-MiSFqAqKH1M,1398
 oafuncs/oa_down/user_agent.py,sha256=LCVQUA60ukUqeJXgLktDHB2sh-ngk7AiX4sKj8w-X4A,416
@@ -39,8 +39,8 @@ oafuncs/oa_sign/__init__.py,sha256=JSx1fcWpmNhQBvX_Bmq3xysfSkkFMrjbJASxV_V6aqE,1
 oafuncs/oa_sign/meteorological.py,sha256=3MSjy7HTcvz2zsITkjUMr_0Y027Gas1LFE9pk99990k,6110
 oafuncs/oa_sign/ocean.py,sha256=3uYEzaq-27yVy23IQoqy-clhWu1I_fhPFBAQyT-OF4M,5562
 oafuncs/oa_sign/scientific.py,sha256=moIl2MEY4uitbXoD596JmXookXGQtQsS-8_1NBBTx84,4689
-oafuncs-0.0.98.46.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.98.46.dist-info/METADATA,sha256=ZeQYycohu3zboTLafN-CHlEwkhmixvmazKaLAADhFpI,4446
-oafuncs-0.0.98.46.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-oafuncs-0.0.98.46.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.98.46.dist-info/RECORD,,
+oafuncs-0.0.98.47.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.98.47.dist-info/METADATA,sha256=iJT9zUhIKuQGVu3zNAoQvKq9DyB6oMm-mlptyweqRR0,4446
+oafuncs-0.0.98.47.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+oafuncs-0.0.98.47.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.98.47.dist-info/RECORD,,