oafuncs 0.0.98.46__tar.gz → 0.0.98.49__tar.gz

{oafuncs-0.0.98.46/oafuncs.egg-info → oafuncs-0.0.98.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.46
+Version: 0.0.98.49
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.46 → oafuncs-0.0.98.49}/oafuncs/oa_down/literature.py

@@ -12,7 +12,7 @@ from oafuncs.oa_down.user_agent import get_ua
 from oafuncs.oa_file import remove
 from oafuncs.oa_data import ensure_list
 
-__all__ = ["download5doi", "download5doi_via_unpaywall"]
+__all__ = ["download5doi"]
 
 
 def _get_file_size(file_path, unit="KB"):
@@ -368,129 +368,6 @@ def download5doi(
         dl.download_pdf()
 
 
-# ------------------------------- 合规替代方案（Open Access 优先） -------------------------------
-def _get_oa_pdf_url_from_unpaywall(doi: str, email: str | None) -> str | None:
-    """
-    通过 Unpaywall 获取可开放访问的 PDF 链接（若存在）。
-    需要提供 email（Unpaywall 要求标识邮件）。
-    返回 PDF URL 或 None。
-    """
-    if not email:
-        print("[bold yellow]Unpaywall 需要 email 参数；请提供 email 以查询 OA 链接。")
-        return None
-    api = f"https://api.unpaywall.org/v2/{doi}?email={email}"
-    try:
-        r = requests.get(api, timeout=15)
-        if r.status_code != 200:
-            print(f"[bold yellow]Unpaywall 查询失败: HTTP {r.status_code}")
-            return None
-        data = r.json()
-        loc = data.get("best_oa_location") or {}
-        url_for_pdf = loc.get("url_for_pdf") or loc.get("url")
-        if url_for_pdf and url_for_pdf.lower().endswith(".pdf"):
-            return url_for_pdf
-        # 有些 OA 链接是落在 landing page，再尝试从记录的所有位置挑选 pdf
-        for k in ("oa_locations", "oa_location"):
-            entries = data.get(k) or []
-            if isinstance(entries, dict):
-                entries = [entries]
-            for e in entries:
-                u = e.get("url_for_pdf") or e.get("url")
-                if u and ".pdf" in u.lower():
-                    return u
-    except Exception as e:
-        print(f"[bold yellow]Unpaywall 查询异常: {e}")
-    return None
-
-
-def _download_pdf_from_url(url: str, dest_path: Path, headers: dict | None = None) -> bool:
-    """
-    给定合法的 PDF 下载 URL，下载保存到 dest_path。
-    返回 True/False 表示是否成功。
-    """
-    headers = headers or {"User-Agent": str(get_ua()), "Accept": "application/pdf"}
-    try:
-        with requests.get(url, headers=headers, stream=True, timeout=30) as r:
-            if r.status_code != 200 or "application/pdf" not in r.headers.get("Content-Type", "").lower():
-                # 仍可能是 PDF（某些服务器未正确设置头），尝试保存但标注提示
-                if r.status_code != 200:
-                    print(f"[bold yellow]下载失败: HTTP {r.status_code}")
-                    return False
-            dest_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(dest_path, "wb") as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    if chunk:
-                        f.write(chunk)
-        return True
-    except Exception as e:
-        print(f"[bold yellow]下载异常: {e}")
-        return False
-
-
-def download5doi_via_unpaywall(
-    store_path=None,
-    doi_list=None,
-    txt_file=None,
-    excel_file=None,
-    col_name=r"DOI",
-    email: str | None = None,
-):
-    """
-    优先使用 Unpaywall 获取开放访问（OA）的 PDF 并下载，避免非合规站点。
-
-    参数：
-        store_path: 保存目录
-        doi_list/txt_file/excel_file/col_name: 同 download5doi
-        email: 用于访问 Unpaywall API 的邮箱（必填，否则无法查询）
-    """
-    if not store_path:
-        store_path = Path.cwd()
-    else:
-        store_path = Path(str(store_path))
-    store_path.mkdir(parents=True, exist_ok=True)
-
-    if doi_list:
-        doi_list = ensure_list(doi_list)
-    if txt_file:
-        doi_list = _read_txt(txt_file)
-    if excel_file:
-        doi_list = _read_excel(excel_file, col_name)
-
-    if not doi_list:
-        print("[bold yellow]未提供 DOI 列表。")
-        return
-
-    print(f"[bold cyan]通过 Unpaywall 尝试下载 {len(doi_list)} 篇 OA PDF...")
-    ok, miss = 0, 0
-    for doi in track(doi_list, description="OA downloading..."):
-        # 规范化文件名
-        fname = re.sub(r'[/<>:"?*|]', "_", str(doi)) + ".pdf"
-        dest = store_path / fname
-        if dest.exists() and _get_file_size(dest, unit="KB") > 10:
-            ok += 1
-            continue
-
-        pdf_url = _get_oa_pdf_url_from_unpaywall(str(doi), email=email)
-        if not pdf_url:
-            miss += 1
-            print(f"[bold yellow]未找到 OA PDF: {doi}")
-            continue
-
-        if _download_pdf_from_url(pdf_url, dest):
-            size_kb = _get_file_size(dest, unit="KB")
-            if isinstance(size_kb, (int, float)) and size_kb < 10:
-                dest.unlink(missing_ok=True)
-                miss += 1
-                print(f"[bold yellow]文件过小，疑似异常，已删除: {dest}")
-            else:
-                ok += 1
-                print(f"[bold green]已下载: {dest}")
-        else:
-            miss += 1
-
-    print(f"[bold]完成。成功 {ok} 篇，未获取 {miss} 篇（可能无 OA 版本或需机构访问）。")
-
-
 if __name__ == "__main__":
     store_path = r"F:\AAA-Delete\DOI_Reference\5\pdf"
     excel_file = r"F:\AAA-Delete\DOI_Reference\5\savedrecs.xls"

{oafuncs-0.0.98.46 → oafuncs-0.0.98.49}/oafuncs/oa_linux.py

@@ -54,10 +54,15 @@ def query_queue(need_node=1, queue_list =['dcu','bigmem','cpu_parallel','cpu_sin
     queue_dict = get_queue_node()
     hs = None
     for my_queue in queue_list:
-        if my_queue in queue_dict and queue_dict[my_queue] >= need_node:
-            # slurm_file = f'../run.slurm.{my_queue}'
-            hs = my_queue
-            break
+        if my_queue == 'cpu_parallel':
+            for mq in ['cpu_parallel','cpu_parallel*']:
+                if mq in queue_dict and queue_dict[mq] >= need_node:
+                    hs = 'cpu_parallel'
+                    break
+        else:
+            if my_queue in queue_dict and queue_dict[my_queue] >= need_node:
+                hs = my_queue
+                break
     return hs
 
 def running_jobs():
@@ -68,11 +73,18 @@ def running_jobs():
     ids = [job.split()[0] for job in Jobs if job != '']
     return ids
 
-def submit_job(working_dir, script_tmp='run.slurm', script_run='run.slurm', need_node=1, queue_tmp='<queue_name>', queue_list=['dcu', 'bigmem', 'cpu_parallel', 'cpu_single'], max_job=38):
+def submit_job(working_dir=None, script_tmp='run.slurm', script_run='run.slurm', need_node=1, queue_tmp='<queue_name>', queue_list=['dcu', 'bigmem', 'cpu_parallel', 'cpu_single'], max_job=38, wait=False): 
+    '''提交任务到集群，并返回任务ID'''
     from .oa_file import replace_content
     import datetime
+    if working_dir is None:
+        working_dir = os.getcwd()
     os.chdir(working_dir)
     print(f'切换工作目录到: {working_dir}')
+    
+    if need_node > 1 and 'cpu_single' in queue_list:
+        queue_list.remove('cpu_single')
+    
     while True:
         running_job = running_jobs()
         if not running_job or len(running_job) < max_job:
@@ -94,6 +106,7 @@ def submit_job(working_dir, script_tmp='run.slurm', script_run='run.slurm', need
                         time.sleep(30)
                     else:
                         print(f'提交任务成功，{content_sub.strip()}')
+                        job_id = content_sub.strip().split()[-1]
                         break
             else:
                 print('没有足够的计算资源，等待30秒后重试！')
@@ -103,6 +116,20 @@ def submit_job(working_dir, script_tmp='run.slurm', script_run='run.slurm', need
             time.sleep(60)
     print(f'等待10秒后，继续检查任务状态！')
     time.sleep(10)
+    
+    if wait:
+        while True:
+            if job_id in running_jobs():
+                print(f'Time: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
+                print(f'任务{job_id}正在队列中...')
+                time.sleep(60)
+            else:
+                print(f'任务{job_id}已完成！')
+                break
+    else:
+        print(f'任务{job_id}已提交，不等待其完成，继续执行后续操作！')
+    
+    return job_id
 
 if __name__ == "__main__":
     pass

{oafuncs-0.0.98.46 → oafuncs-0.0.98.49/oafuncs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.46
+Version: 0.0.98.49
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.46 → oafuncs-0.0.98.49}/setup.py

@@ -18,7 +18,7 @@ URL = "https://github.com/Industry-Pays/OAFuncs"
 EMAIL = "liukun0312@stu.ouc.edu.cn"
 AUTHOR = "Kun Liu"
 REQUIRES_PYTHON = ">=3.10.0"  # 2025/03/13
-VERSION = "0.0.98.46"
+VERSION = "0.0.98.49"
 
 # What packages are required for this module to be executed?
 REQUIRED = [