funread 1.1.2__tar.gz → 1.1.4__tar.gz

{funread-1.1.2/src/funread.egg-info → funread-1.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funread
-Version: 1.1.2
+Version: 1.1.4
 Summary: Add your description here
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown

{funread-1.1.2 → funread-1.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "funread"
-version = "1.1.2"
+version = "1.1.4"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.8"

{funread-1.1.2 → funread-1.1.4}/src/funread/legado/manage/download/base.py

@@ -1,15 +1,19 @@
 import json
 import os
-from datetime import datetime
 import traceback
+from datetime import datetime
+
 import pandas as pd
 import requests
 from funfile import pickle, funos
 from funfile.compress import tarfile
 from funread.legado.manage.utils import url_to_hostname
 from funsecret import get_md5_str
+from funutil import getLogger
 from tqdm import tqdm
 
+logger = getLogger('funread')
+
 
 class DownloadSource(object):
     def __init__(self, path="./funread-hub", cate1="rss", *args, **kwargs):
@@ -29,26 +33,11 @@ class DownloadSource(object):
         funos.makedirs(self.path_bok)
         funos.makedirs(self.path_pkl)
 
-    def loads(self):
-        print("loads")
-        if os.path.exists(self.pkl_url):
-            df = pd.read_pickle(self.pkl_url, compression="infer")
-            self.url_map = {k: v for k, v in df.values}
-        else:
-            self.url_map = {"https://farfarfun.github.com": 100000}
-        self.current_id = max(self.url_map.values())
-        if os.path.exists(self.pkl_md5):
-            df = pd.read_pickle(self.pkl_md5, compression="infer")
-            self.md5_set = {info["md5"]: info for info in df.to_dict(orient="records")}
-
-    def dumps(self):
-        print("dumps")
+    def loader(self):
+        raise NotImplementedError
 
-        df = pd.DataFrame([{"url": k, "url_id": v} for k, v in self.url_map.items()])
-        df.to_pickle(self.pkl_url, compression="infer")
-
-        df = pd.DataFrame(self.md5_set.values())
-        df.to_pickle(self.pkl_md5, compression="infer")
+    def source_format(self, source):
+        raise NotImplementedError
 
     def url_index(self, url):
         if url in self.url_map:
@@ -58,8 +47,30 @@ class DownloadSource(object):
             self.url_map[url] = self.current_id
             return self.current_id
 
-    def source_format(self, source):
-        pass
+    @staticmethod
+    def add_source_to_candidate(md5, fpath, source, url_info=None):
+        url_info = url_info or {}
+        if os.path.exists(fpath):
+            data = json.loads(open(fpath, "r").read())
+        else:
+            data = {
+                "available": True,
+                "merged": [],
+                "candidate": [],
+                "url_id": url_info["url_id"],
+                "hostname": url_info["hostname"],
+            }
+        md5_list = []
+        [
+            [md5_list.extend(md5["md5_list"]) for md5 in data[key]]
+            for key in ("merged", "candidate")
+            if key in data
+        ]
+        if md5 not in md5_list:
+            data["candidate"].append({"md5_list": [md5], "source": source})
+
+        with open(fpath, "w") as fw:
+            fw.write(json.dumps(data, sort_keys=True, indent=4))
 
     def add_source(self, source, *args, **kwargs):
         md5 = get_md5_str(json.dumps(source))
@@ -99,33 +110,7 @@ class DownloadSource(object):
             try:
                 self.add_source(source, *args, **kwargs)
             except Exception as e:
-                # traceback.print_exc()
-                # print(e)
-                pass
-
-    def add_source_to_candidate(self, md5, fpath, source, url_info=None):
-        url_info = url_info or {}
-        if os.path.exists(fpath):
-            data = json.loads(open(fpath, "r").read())
-        else:
-            data = {
-                "available": True,
-                "merged": [],
-                "candidate": [],
-                "url_id": url_info["url_id"],
-                "hostname": url_info["hostname"],
-            }
-        md5_list = []
-        [
-            [md5_list.extend(md5["md5_list"]) for md5 in data[key]]
-            for key in ("merged", "candidate")
-            if key in data
-        ]
-        if md5 not in md5_list:
-            data["candidate"].append({"md5_list": [md5], "source": source})
-
-        with open(fpath, "w") as fw:
-            fw.write(json.dumps(data, sort_keys=True, indent=4))
+                logger.error(f"error: {e},traceback: {traceback.format_exc()}")
 
     def export_sources(self, size=1000):
         file_list = []
@@ -151,6 +136,27 @@ class DownloadSource(object):
                         dd = []
         yield dd
 
+    def loads(self):
+        logger.info("loads")
+
+        if os.path.exists(self.pkl_url):
+            df = pd.read_pickle(self.pkl_url, compression="infer")
+            self.url_map = {k: v for k, v in df.values}
+        else:
+            self.url_map = {"https://farfarfun.github.com": 100000}
+        self.current_id = max(self.url_map.values())
+        if os.path.exists(self.pkl_md5):
+            df = pd.read_pickle(self.pkl_md5, compression="infer")
+            self.md5_set = {info["md5"]: info for info in df.to_dict(orient="records")}
+
+    def dumps(self):
+        logger.info("dumps")
+        df = pd.DataFrame([{"url": k, "url_id": v} for k, v in self.url_map.items()])
+        df.to_pickle(self.pkl_url, compression="infer")
+
+        df = pd.DataFrame(self.md5_set.values())
+        df.to_pickle(self.pkl_md5, compression="infer")
+
     def loads_zip(self, zip_file=None):
         funos.delete(self.path_pkl)
         funos.delete(self.path_bok)
@@ -158,7 +164,7 @@ class DownloadSource(object):
             files = os.listdir(self.path_bak)
             files.sort(key=lambda x: x)
             zip_file = os.path.join(self.path_bak, files[-1])
-        print(f"load zip file from {zip_file}")
+        logger.info(f"load zip file from {zip_file}")
         with tarfile.open(zip_file, "r") as tar:
             tar.extractall(self.path_rot)
         self.loads()
@@ -166,6 +172,7 @@ class DownloadSource(object):
     def dumps_zip(self):
         self.dumps()
         zip_file = f"{self.path_bak}/{self.cate1}-{datetime.now().strftime('%Y%m%d-%H%M%S')}.tar.xz"
+        logger.info(f"dump zip file to {zip_file}")
         with tarfile.open(zip_file, "w|xz") as tar:
             tar.add(self.path_pkl, arcname=os.path.basename(self.path_pkl))
             tar.add(self.path_bok, arcname=os.path.basename(self.path_bok))

{funread-1.1.2 → funread-1.1.4}/src/funread/legado/manage/download/rss.py

@@ -3,12 +3,14 @@ import os
 import re
 
 import requests
+from funfake.headers import Headers
 from funfile import funos
-from funread.legado.manage.download.base import DownloadSource
 from funsecret import get_md5_str
 from funutil import getLogger
 from tqdm import tqdm
 
+from funread.legado.manage.download.base import DownloadSource
+
 logger = getLogger("funread")
 
 
@@ -62,6 +64,7 @@ class RSSSourceDownload(DownloadSource):
         return RSSSourceFormat(source).run()
 
     def rss_download(self):
+        faker = Headers()
         urls = [
             "https://jt12.de/SYV2_4/2024/03/04/20/48/54/170955653465e5c33680046.json",
             "https://jt12.de/SYV2/2023/03/17/0/02/42/167898256264133da2869d4.json",
@@ -81,7 +84,7 @@ class RSSSourceDownload(DownloadSource):
             if index > size and os.path.exists(file):
                 continue
             try:
-                data = json.dumps(requests.get(url).json())
+                data = json.dumps(requests.get(url, headers=faker.generate()).json())
                 with open(file, "w") as fw:
                     fw.write(data)
             except Exception as e:

funread-1.1.4/src/funread/legado/manage/rss.py

@@ -0,0 +1,93 @@
+from datetime import datetime
+
+import requests
+from fundrive import GithubDrive
+from funfake.headers import Headers
+from funtask import Task
+
+faker = Headers()
+
+
+class UpdateRssTask(Task):
+    def __init__(self):
+        self.drive = GithubDrive()
+        self.drive.login("farfarfun/funread-cache")
+        super(UpdateRssTask, self).__init__()
+
+    def random_icon(self):
+        url = "https://api.thecatapi.com/v1/images/search?size=full"
+        response = requests.get(url, headers=faker.generate()).json()
+        return response[0]["url"]
+
+    def update_book(self, dir_path="funread/legado/snapshot/lasted"):
+        dl = []
+        for dir_info in self.drive.get_dir_list(dir_path):
+            dl.append(
+                {
+                    "title": dir_info["name"],
+                    "pic": self.random_icon(),
+                    "url": f"https://farfarfun.github.io/funread-cache/{dir_info['path']}/index.html",
+                    "description": "this is content",
+                }
+            )
+
+        dl.append(
+            {
+                "title": "源仓库(新)",
+                "url": "https://link3.cc/yckceo",
+            }
+        )
+
+        dl.append({"title": "开源阅读-语雀文档", "url": "https://www.yuque.com/legado"})
+        dl.append({"title": "喵公子书源", "url": "http://yuedu.miaogongzi.net/gx.html"})
+        dl.append({"title": "「阅读」APP 源-aoaostar", "url": "https://legado.aoaostar.com/"})
+        dl.append({"title": "yiove", "url": "https://shuyuan.yiove.com/"})
+
+        #
+        for line in dl:
+            if "pic" not in line:
+                line["pic"] = self.random_icon()
+            if "time" not in line:
+                line["time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        data1 = {
+            "name": "test",
+            "next": "https://gitee.com/farfarfun/funread-cache/raw/master/funread/legado/snapshot/lasted/funread.json",
+            "list": dl,
+        }
+        self.drive.upload_file(git_path=f"{dir_path}/source.json", content=data1)
+
+    def update_main(self):
+        rss = {
+            "源": "https://gitee.com/farfarfun/funread-cache/raw/master/funread/legado/snapshot/lasted/source.json",
+            "源(备)": "https://github.com/farfarfun/funread-cache/raw/master/funread/legado/snapshot/lasted/source.json",
+        }
+
+        data2 = [
+            {
+                "lastUpdateTime": int(datetime.now().timestamp()),
+                "sourceName": "funread",
+                "sourceIcon": self.random_icon(),
+                "sourceUrl": "https://github.com/farfarfun",
+                "loadWithBaseUrl": False,
+                "singleUrl": False,
+                "sortUrl": "\n".join([f"{k}::{v}" for k, v in rss.items()]),
+                "ruleArticles": "$.list[*]",
+                "ruleNextArticles": "$.next",
+                "ruleTitle": "$.title",
+                "rulePubDate": "⏰{{$.time}}",
+                "ruleImage": "$.pic",
+                "ruleLink": "$.url",
+                "sourceGroup": "VIP",
+                "customOrder": -9999999,
+                "enabled": True,
+            }
+        ]
+        self.drive.upload_file(git_path="funread/legado/snapshot/lasted/funread.json", content=data2)
+
+    def run(self):
+        self.update_book()
+        self.update_main()
+
+
+UpdateRssTask().run()

{funread-1.1.2 → funread-1.1.4/src/funread.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funread
-Version: 1.1.2
+Version: 1.1.4
 Summary: Add your description here
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown

{funread-1.1.2 → funread-1.1.4}/src/funread.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ src/funread.egg-info/requires.txt
 src/funread.egg-info/top_level.txt
 src/funread/legado/__init__.py
 src/funread/legado/manage/__init__.py
+src/funread/legado/manage/rss.py
 src/funread/legado/manage/download/__init__.py
 src/funread/legado/manage/download/base.py
 src/funread/legado/manage/download/book.py