funread 1.0.1__tar.gz

funread-1.0.1/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1.  Definitions.
+
+    "License" shall mean the terms and conditions for use, reproduction,
+    and distribution as defined by Sections 1 through 9 of this document.
+
+    "Licensor" shall mean the copyright owner or entity authorized by
+    the copyright owner that is granting the License.
+
+    "Legal Entity" shall mean the union of the acting entity and all
+    other entities that control, are controlled by, or are under common
+    control with that entity. For the purposes of this definition,
+    "control" means (i) the power, direct or indirect, to cause the
+    direction or management of such entity, whether by contract or
+    otherwise, or (ii) ownership of fifty percent (50%) or more of the
+    outstanding shares, or (iii) beneficial ownership of such entity.
+
+    "You" (or "Your") shall mean an individual or Legal Entity
+    exercising permissions granted by this License.
+
+    "Source" form shall mean the preferred form for making modifications,
+    including but not limited to software source code, documentation
+    source, and configuration files.
+
+    "Object" form shall mean any form resulting from mechanical
+    transformation or translation of a Source form, including but
+    not limited to compiled object code, generated documentation,
+    and conversions to other media types.
+
+    "Work" shall mean the work of authorship, whether in Source or
+    Object form, made available under the License, as indicated by a
+    copyright notice that is included in or attached to the work
+    (an example is provided in the Appendix below).
+
+    "Derivative Works" shall mean any work, whether in Source or Object
+    form, that is based on (or derived from) the Work and for which the
+    editorial revisions, annotations, elaborations, or other modifications
+    represent, as a whole, an original work of authorship. For the purposes
+    of this License, Derivative Works shall not include works that remain
+    separable from, or merely link (or bind by name) to the interfaces of,
+    the Work and Derivative Works thereof.
+
+    "Contribution" shall mean any work of authorship, including
+    the original version of the Work and any modifications or additions
+    to that Work or Derivative Works thereof, that is intentionally
+    submitted to Licensor for inclusion in the Work by the copyright owner
+    or by an individual or Legal Entity authorized to submit on behalf of
+    the copyright owner. For the purposes of this definition, "submitted"
+    means any form of electronic, verbal, or written communication sent
+    to the Licensor or its representatives, including but not limited to
+    communication on electronic mailing lists, source code control systems,
+    and issue tracking systems that are managed by, or on behalf of, the
+    Licensor for the purpose of discussing and improving the Work, but
+    excluding communication that is conspicuously marked or otherwise
+    designated in writing by the copyright owner as "Not a Contribution."
+
+    "Contributor" shall mean Licensor and any individual or Legal Entity
+    on behalf of whom a Contribution has been received by Licensor and
+    subsequently incorporated within the Work.
+
+2.  Grant of Copyright License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    copyright license to reproduce, prepare Derivative Works of,
+    publicly display, publicly perform, sublicense, and distribute the
+    Work and such Derivative Works in Source or Object form.
+
+3.  Grant of Patent License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    (except as stated in this section) patent license to make, have made,
+    use, offer to sell, sell, import, and otherwise transfer the Work,
+    where such license applies only to those patent claims licensable
+    by such Contributor that are necessarily infringed by their
+    Contribution(s) alone or by combination of their Contribution(s)
+    with the Work to which such Contribution(s) was submitted. If You
+    institute patent litigation against any entity (including a
+    cross-claim or counterclaim in a lawsuit) alleging that the Work
+    or a Contribution incorporated within the Work constitutes direct
+    or contributory patent infringement, then any patent licenses
+    granted to You under this License for that Work shall terminate
+    as of the date such litigation is filed.
+
+4.  Redistribution. You may reproduce and distribute copies of the
+    Work or Derivative Works thereof in any medium, with or without
+    modifications, and in Source or Object form, provided that You
+    meet the following conditions:
+
+    (a) You must give any other recipients of the Work or
+    Derivative Works a copy of this License; and
+
+    (b) You must cause any modified files to carry prominent notices
+    stating that You changed the files; and
+
+    (c) You must retain, in the Source form of any Derivative Works
+    that You distribute, all copyright, patent, trademark, and
+    attribution notices from the Source form of the Work,
+    excluding those notices that do not pertain to any part of
+    the Derivative Works; and
+
+    (d) If the Work includes a "NOTICE" text file as part of its
+    distribution, then any Derivative Works that You distribute must
+    include a readable copy of the attribution notices contained
+    within such NOTICE file, excluding those notices that do not
+    pertain to any part of the Derivative Works, in at least one
+    of the following places: within a NOTICE text file distributed
+    as part of the Derivative Works; within the Source form or
+    documentation, if provided along with the Derivative Works; or,
+    within a display generated by the Derivative Works, if and
+    wherever such third-party notices normally appear. The contents
+    of the NOTICE file are for informational purposes only and
+    do not modify the License. You may add Your own attribution
+    notices within Derivative Works that You distribute, alongside
+    or as an addendum to the NOTICE text from the Work, provided
+    that such additional attribution notices cannot be construed
+    as modifying the License.
+
+    You may add Your own copyright statement to Your modifications and
+    may provide additional or different license terms and conditions
+    for use, reproduction, or distribution of Your modifications, or
+    for any such Derivative Works as a whole, provided Your use,
+    reproduction, and distribution of the Work otherwise complies with
+    the conditions stated in this License.
+
+5.  Submission of Contributions. Unless You explicitly state otherwise,
+    any Contribution intentionally submitted for inclusion in the Work
+    by You to the Licensor shall be under the terms and conditions of
+    this License, without any additional terms or conditions.
+    Notwithstanding the above, nothing herein shall supersede or modify
+    the terms of any separate license agreement you may have executed
+    with Licensor regarding such Contributions.
+
+6.  Trademarks. This License does not grant permission to use the trade
+    names, trademarks, service marks, or product names of the Licensor,
+    except as required for reasonable and customary use in describing the
+    origin of the Work and reproducing the content of the NOTICE file.
+
+7.  Disclaimer of Warranty. Unless required by applicable law or
+    agreed to in writing, Licensor provides the Work (and each
+    Contributor provides its Contributions) on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+    implied, including, without limitation, any warranties or conditions
+    of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+    PARTICULAR PURPOSE. You are solely responsible for determining the
+    appropriateness of using or redistributing the Work and assume any
+    risks associated with Your exercise of permissions under this License.
+
+8.  Limitation of Liability. In no event and under no legal theory,
+    whether in tort (including negligence), contract, or otherwise,
+    unless required by applicable law (such as deliberate and grossly
+    negligent acts) or agreed to in writing, shall any Contributor be
+    liable to You for damages, including any direct, indirect, special,
+    incidental, or consequential damages of any character arising as a
+    result of this License or out of the use or inability to use the
+    Work (including but not limited to damages for loss of goodwill,
+    work stoppage, computer failure or malfunction, or any and all
+    other commercial damages or losses), even if such Contributor
+    has been advised of the possibility of such damages.
+
+9.  Accepting Warranty or Additional Liability. While redistributing
+    the Work or Derivative Works thereof, You may choose to offer,
+    and charge a fee for, acceptance of support, warranty, indemnity,
+    or other liability obligations and/or rights consistent with this
+    License. However, in accepting such obligations, You may act only
+    on Your own behalf and on Your sole responsibility, not on behalf
+    of any other Contributor, and only if You agree to indemnify,
+    defend, and hold each Contributor harmless for any liability
+    incurred by, or claims asserted against, such Contributor by reason
+    of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+Copyright [2023] [niuliangtao]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

funread-1.0.1/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: funread
+Version: 1.0.1
+Summary: 
+Author: farfun
+Author-email: 1007530194@qq.com
+Requires-Python: >=3.8,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Description-Content-Type: text/markdown
+
+# 阅读
+
+[源仓库](http://www.yckceo1.com/)
+
+[规则说明](https://alanskycn.gitee.io/teachme/)
+
+[app端](https://github.com/gedoor/legado)
+
+[服务端](https://github.com/hectorqin/reader)
+

funread-1.0.1/README.md

@@ -0,0 +1,9 @@
+# 阅读
+
+[源仓库](http://www.yckceo1.com/)
+
+[规则说明](https://alanskycn.gitee.io/teachme/)
+
+[app端](https://github.com/gedoor/legado)
+
+[服务端](https://github.com/hectorqin/reader)

funread-1.0.1/pyproject.toml

@@ -0,0 +1,16 @@
+[build-system]
+requires = [ "poetry-core",]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "funread"
+version = "1.0.1"
+description = ""
+authors = [ "farfun <1007530194@qq.com>",]
+readme = "README.md"
+[[tool.poetry.packages]]
+from = "src"
+include = "*"
+
+[tool.poetry.dependencies]
+python = "^3.8"

funread-1.0.1/src/funread/legado/manage/download/__init__.py

@@ -0,0 +1,4 @@
+from .base import DownloadSource
+from .book import BookSourceFormat,BookSourceDownload
+from.generate import GenerateSourceTask
+from .rss import RSSSourceDownload

funread-1.0.1/src/funread/legado/manage/download/base.py

@@ -0,0 +1,178 @@
+import json
+import os
+from datetime import datetime
+import traceback
+import pandas as pd
+import requests
+from funfile import pickle, funos
+from funfile.compress import tarfile
+from funread.legado.manage.utils import url_to_hostname
+from funsecret import get_md5_str
+from tqdm import tqdm
+
+
+class DownloadSource(object):
+    def __init__(self, path="./funread-hub", cate1="rss", *args, **kwargs):
+        self.cate1 = cate1
+        self.path_rot = f"{path}/{cate1}"
+        self.path_bak = f"{path}/{cate1}/bak"
+        self.path_pkl = f"{path}/{cate1}/pkl"
+        self.path_bok = f"{path}/{cate1}/source"
+        self.pkl_url = f"{self.path_pkl}/url_info.pkl.bz2"
+        self.pkl_md5 = f"{self.path_pkl}/source_info.pkl.bz2"
+
+        self.url_map = {}
+        self.md5_set = {}
+        self.current_id = 1
+
+        funos.makedirs(self.path_bak)
+        funos.makedirs(self.path_bok)
+        funos.makedirs(self.path_pkl)
+
+    def loads(self):
+        print("loads")
+        if os.path.exists(self.pkl_url):
+            df = pd.read_pickle(self.pkl_url, compression="infer")
+            self.url_map = {k: v for k, v in df.values}
+        else:
+            self.url_map = {"https://farfarfun.github.com": 100000}
+        self.current_id = max(self.url_map.values())
+        if os.path.exists(self.pkl_md5):
+            df = pd.read_pickle(self.pkl_md5, compression="infer")
+            self.md5_set = {info["md5"]: info for info in df.to_dict(orient="records")}
+
+    def dumps(self):
+        print("dumps")
+
+        df = pd.DataFrame([{"url": k, "url_id": v} for k, v in self.url_map.items()])
+        df.to_pickle(self.pkl_url, compression="infer")
+
+        df = pd.DataFrame(self.md5_set.values())
+        df.to_pickle(self.pkl_md5, compression="infer")
+
+    def url_index(self, url):
+        if url in self.url_map:
+            return self.url_map[url]
+        else:
+            self.current_id += 1
+            self.url_map[url] = self.current_id
+            return self.current_id
+
+    def source_format(self, source):
+        pass
+
+    def add_source(self, source, *args, **kwargs):
+        md5 = get_md5_str(json.dumps(source))
+        source = self.source_format(source)
+        hostname = url_to_hostname(source["sourceUrl"])
+        if hostname is None:
+            return
+        url_id = self.url_index(hostname)
+
+        cate1 = (url_id // 100) * 100
+        fdir = f"{self.path_bok}/{cate1}-{cate1 + 100}/"
+        if not os.path.exists(fdir):
+            os.makedirs(fdir)
+        fpath = f"{fdir}/{url_id}.json"
+
+        url_info = {"url_id": url_id, "hostname": hostname, "cate1": cate1}
+        self.add_source_to_candidate(md5, fpath, source, url_info=url_info)
+
+        self.md5_set[md5] = {
+            "md5": md5,
+            "url_id": url_id,
+            "hostname": hostname,
+            "cate1": cate1,
+        }
+
+    def add_sources(self, data, *args, **kwargs):
+        """"""
+        if isinstance(data, str):
+            if data.startswith("http"):
+                data = requests.get(data).json()
+            elif os.path.exists(data):
+                data = pickle.load(data)
+            elif data[0] == "[" or data[0] == "{":
+                data = json.loads(data)
+
+        for source in tqdm(data):
+            try:
+                self.add_source(source, *args, **kwargs)
+            except Exception as e:
+                # traceback.print_exc()
+                # print(e)
+                pass
+
+    def add_source_to_candidate(self, md5, fpath, source, url_info=None):
+        url_info = url_info or {}
+        if os.path.exists(fpath):
+            data = json.loads(open(fpath, "r").read())
+        else:
+            data = {
+                "available": True,
+                "merged": [],
+                "candidate": [],
+                "url_id": url_info["url_id"],
+                "hostname": url_info["hostname"],
+            }
+        md5_list = []
+        [
+            [md5_list.extend(md5["md5_list"]) for md5 in data[key]]
+            for key in ("merged", "candidate")
+            if key in data
+        ]
+        if md5 not in md5_list:
+            data["candidate"].append({"md5_list": [md5], "source": source})
+
+        with open(fpath, "w") as fw:
+            fw.write(json.dumps(data, sort_keys=True, indent=4))
+
+    def export_sources(self, size=1000):
+        file_list = []
+        for root, dirs, files in os.walk(self.path_bok):
+            for file in files:
+                if file.endswith(".json"):
+                    file_list.append(os.path.join(root, file))
+        dd = []
+        for file in tqdm(file_list):
+            data = open(file, "r").read()
+            data = json.loads(data)
+            if not data.get("available", True):
+                continue
+            for key in ("candidate", "merged"):
+                for item in data[key][:3]:
+                    source = item["source"]
+                    source["sourceUrl"] = (
+                        f"{source['sourceUrl']}#{item['md5_list'][0][:10]}"
+                    )
+                    dd.append(source)
+                    if len(dd) >= size:
+                        yield dd
+                        dd = []
+        yield dd
+
+    def loads_zip(self, zip_file=None):
+        funos.delete(self.path_pkl)
+        funos.delete(self.path_bok)
+        if zip_file is None:
+            files = os.listdir(self.path_bak)
+            files.sort(key=lambda x: x)
+            zip_file = os.path.join(self.path_bak, files[-1])
+        print(f"load zip file from {zip_file}")
+        with tarfile.open(zip_file, "r") as tar:
+            tar.extractall(self.path_rot)
+        self.loads()
+
+    def dumps_zip(self):
+        self.dumps()
+        zip_file = f"{self.path_bak}/{self.cate1}-{datetime.now().strftime('%Y%m%d-%H%M%S')}.tar.xz"
+        with tarfile.open(zip_file, "w|xz") as tar:
+            tar.add(self.path_pkl, arcname=os.path.basename(self.path_pkl))
+            tar.add(self.path_bok, arcname=os.path.basename(self.path_bok))
+
+    def __enter__(self):
+        self.loads()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.dumps()

funread-1.0.1/src/funread/legado/manage/download/book.py

@@ -0,0 +1,121 @@
+import re
+
+from funread.legado.manage.download.base import DownloadSource
+
+
+def retain_zh_ch_dig(text):
+    return re.sub("[^\u4e00-\u9fa5a-zA-Z0-9\[\]]+", "", text)
+
+
+class BookSourceFormat:
+    def __init__(self, source):
+
+        self.source = source
+        self.source["bookSourceComment"] = ""
+        self.source["bookSourceUrl"] = self.source["bookSourceUrl"].rstrip("/|#")
+        if "httpUserAgent" in self.source.keys():
+            self.source["header"] = self.source.pop("httpUserAgent")
+
+        for key in ["bookSourceGroup", "bookSourceName"]:
+            self.source[key] = retain_zh_ch_dig(self.source.get(key, ""))
+
+    def run(self):
+        self.format_book_info()
+        self.format_content()
+        self.format_search()
+        self.format_explore()
+        self.format_toc()
+        keys = [key for key in self.source.keys() if not self.source[key]]
+        for key in keys:
+                self.source.pop(key)
+
+        for key in ['customOrder','respondTime','lastUpdateTime']:
+            self.source.pop(key)
+
+        for key in ['searchUrl','exploreUrl']:
+            if key in self.source.keys():
+                self.source[key]=self.source[key].replace(self.source['bookSourceUrl'],'')
+        return self.source
+
+    def __format_base(self, group, map):
+        book_info = self.source.get(group, {})
+
+        for key, name in map.items():
+            if key not in self.source:
+                continue
+            value = self.source.pop(key)
+            if value:
+                book_info[name] = value
+        if len(book_info) > 0:
+            self.source[group] = book_info
+
+    def format_book_info(self):
+        map = {
+            "ruleBookAuthor": "author",
+            "ruleBookContent": "content",
+            "ruleBookContentReplace": "contentReplace",
+            "ruleBookInfoInit": "init",
+            "ruleBookKind": "kind",
+            "ruleBookLastChapter": "lastChapter",
+            "ruleBookName": "name",
+            "ruleBookUrlPattern": "urlPattern",
+            "ruleBookWordCount": "wordCount",
+        }
+        self.__format_base("ruleBookInfo", map)
+
+    def format_content(self):
+        map = {
+            "ruleContentUrl": "url",
+            "ruleContentUrlNext": "urlNext",
+            "ruleBookContentReplaceRegex": "replaceRegex",
+            "ruleBookContentSourceRegex": "sourceRegex",
+            "ruleBookContentWebJs": "webJs",
+        }
+        self.__format_base("ruleContent", map)
+
+    def format_search(self):
+        map = {
+            "ruleSearchUrl": "url",
+            "ruleSearchName": "name",
+            "ruleSearchAuthor": "author",
+            "ruleSearchList": "bookList",
+            "ruleSearchCoverUrl": "coverUrl",
+            "ruleSearchIntroduce": "intro",
+            "ruleSearchKind": "kind",
+            "ruleSearchLastChapter": "lastChapter",
+            "ruleSearchNoteUrl": "noteUrl",
+            "ruleSearchWordCount": "wordCount",
+        }
+        self.__format_base("ruleSearch", map)
+
+    def format_explore(self):
+        map = {
+            "ruleFindUrl": "url",
+            "ruleFindName": "name",
+            "ruleFindAuthor": "author",
+            "ruleFindList": "bookList",
+            "ruleFindCoverUrl": "coverUrl",
+            "ruleFindIntroduce": "intro",
+            "ruleFindKind": "kind",
+            "ruleFindLastChapter": "lastChapter",
+            "ruleFindNoteUrl": "noteUrl",
+        }
+        self.__format_base("ruleExplore", map)
+
+    def format_toc(self):
+        map = {
+            "ruleChapterList": "chapterList",
+            "ruleChapterName": "chapterName",
+            "ruleChapterUpdateTime": "updateTime",
+            "ruleChapterUrl": "chapterUrl",
+            "ruleChapterUrlNext": "nextTocUrl",
+        }
+        self.__format_base("ruleToc", map)
+
+
+class BookSourceDownload(DownloadSource):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def source_format(self, source):
+        return BookSourceFormat(source).run()

funread-1.0.1/src/funread/legado/manage/download/generate.py

@@ -0,0 +1,187 @@
+from datetime import datetime
+
+from dominate.tags import *
+from fundrive import GithubDrive
+from funfile import funos
+from funread.legado.manage.download.rss import RSSSourceDownload
+from funsecret import read_secret
+from funtask import Task
+
+"""
+dominate example: https://blog.csdn.net/wumingxiaoyao/article/details/122894671
+"""
+
+
+class GenerateCommon:
+    def generate_build_repo(self, category, repo):
+        with li(category + ":"):
+            a("github", href=f"https://github.com/{repo}")
+            a("gitee", href=f"https://gitee.com/{repo}")
+            a("gitlink", href=f"https://gitlink.org.cn/{repo}")
+
+    def generate_org(self):
+        build_type_div = div()
+        build_type_fond = b()
+        build_type_fond += font("farfarfun")
+        build_type_div += build_type_fond
+        with ul():
+            self.generate_build_repo("funsecret", "farfarfun/funsecret")
+            self.generate_build_repo("fundrive", "farfarfun/fundrive")
+            self.generate_build_repo("funread", "farfarfun/funread")
+            with li("订阅源:"):
+                a("github",
+                  href="yuedu://rssSource/importonline?src=https://github.com/farfarfun/funread-cache/raw/master/funread/legado/rss/rss-main.json")
+                a("gitee",
+                  href="yuedu://rssSource/importonline?src=https://gitee.com/farfarfun/funread-cache/raw/master/funread/legado/rss/rss-main.json")
+                a("gitlink",
+                  href="yuedu://rssSource/importonline?src=https://gitlink.org.cn/farfarfun/funread-cache/raw/master/funread/legado/rss/rss-main.json")
+
+    def generate(self):
+        br()
+        div(b(font("Build Information", color="#0B610B")))
+        div(hr(size=2, alignment="center", width="100%"))
+        div((b(font("Cause: Started by upstream pipeline job "))))
+        self.generate_org()
+        img(src="https://profile-avatar.csdnimg.cn/2e9aaf5666c044d7aa273e086f9878d0_n1007530194.jpg")
+        br()
+        p("** This is an automatically generated email by jenkins job. **")
+        p("Feel free to connect 1007530194@qq.com if you have any question.")
+        p(f'update time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
+
+
+class GenerateSourceType:
+    style_applied = """
+            body{
+    			font-family: verdana,arial,sans-serif;
+    			font-size:11px;
+    		}
+    		table.gridtable {
+    			color: #333333;
+    			border-width: 1px;
+    			border-color: #666666;
+    			border-collapse: collapse;
+                font-size:11px;
+    		}
+    		table.gridtable th {
+    			border-width: 1px;
+    			padding: 8px;
+    			border-style: solid;
+    			border-color: #666666;
+    			background-color: #DDEBF7;
+    		}
+    		table.gridtable td {
+    			border-width: 1px;
+    			padding: 8px;
+    			border-style: solid;
+    			border-color: #666666;
+    			background-color: #ffffff;
+    			text-align:center;
+    		}
+    		table.gridtable td.failed {
+    			color:#ED5F5F;
+    		}
+    		table.gridtable td.passrate {
+    			font-weight:bold;
+    			color:green;
+    		}
+    		li {
+    			margin-top:5px;
+    		}
+            div{
+                margin-top:10px;
+            }
+        """
+
+    def __init__(self, dir_path="funread/legado/book/snapshot/20231011", source_type="booksource"):
+        self.repo_str = "farfarfun/funread-cache"
+        self.dir_path = dir_path
+        self.source_type = source_type
+        self.drive = GithubDrive()
+        self.drive.login(repo_str=self.repo_str)
+
+    def set_table_head(self):
+        with tr():
+            th("文件")
+            th("大小")
+            th("github")
+            th("gitee")
+
+    def fill_table_data(self, file):
+        data_tr = tr()
+        data_tr += td(file["name"])
+        data_tr += td(file["size"])
+        link_td = td()
+        link_td += a(
+            "github",
+            href=f"yuedu://{self.source_type}/importonline?src=https://github.com/{self.repo_str}/raw/master/{file['path']}",
+        )
+        data_tr += link_td
+        link_td = td()
+        link_td += a(
+            "gitee",
+            href=f"yuedu://{self.source_type}/importonline?src=https://gitee.com/{self.repo_str}/raw/master/{file['path']}",
+        )
+        data_tr += link_td
+
+    def generate_table(self):
+        result_div = div(id="test case result")
+        with result_div.add(table(cls="gridtable")).add(tbody()):
+            self.set_table_head()
+
+            files = self.drive.get_file_list(self.dir_path)
+            for file in files:
+                if file["name"].endswith(".json"):
+                    self.fill_table_data(file)
+
+    def generate_html_report(self):
+        html_root = html()
+        # html head
+        with html_root.add(head()):
+            style(self.style_applied, type="text/css")
+
+        # html body
+        with html_root.add(body()):
+            hello_div = div(id="hello")
+            hello_div.add(p("Hi All,"))
+            hello_div.add(p("This is today's " " API Test Report in "))
+            self.generate_table()
+            GenerateCommon().generate()
+
+        return html_root.render()
+
+    def split_and_upload(self):
+        path = read_secret(cate1='funread', cate2='cache', cate3='path', cate4='root')
+        with RSSSourceDownload(path=path, cate1="rss") as runner:
+            funos.delete(runner.path_pkl)
+            funos.delete(runner.path_bok)
+            runner.loads_zip()
+            i = 1000
+            for data in runner.export_sources(size=3000):
+                if len(data) > 0:
+                    self.drive.upload_file(content=data, git_path=f"{self.dir_path}/progress-{i}.json")
+                    i += 1
+
+    def update_rss(self):
+        print("update source rss")
+        self.drive.upload_file(git_path=f"{self.dir_path}/index.html", content=self.generate_html_report())
+        print(len(self.generate_html_report()))
+
+
+class GenerateSourceTask(Task):
+    def __init__(self, dir_path=f"funread/legado/snapshot/lasted", source_type="booksource", *args, **kwargs):
+        self.repo_str = "farfarfun/funread-cache"
+        self.dir_path = dir_path
+        self.source_type = source_type
+        self.drive = GithubDrive()
+        self.drive.login(repo_str=self.repo_str)
+        super(GenerateSourceTask, self).__init__(*args, **kwargs)
+
+    def run_book(self, *args, **kwargs):
+        generate_source = GenerateSourceType(source_type="booksource", dir_path=f"{self.dir_path}/book")
+        generate_source.split_and_upload()
+        generate_source.update_rss()
+
+    def run_rss(self, *args, **kwargs):
+        generate_source = GenerateSourceType(source_type="rsssource", dir_path=f"{self.dir_path}/rss")
+        generate_source.split_and_upload()
+        generate_source.update_rss()

funread-1.0.1/src/funread/legado/manage/download/rss.py

@@ -0,0 +1,56 @@
+import re
+
+from funread.legado.manage.download.base import DownloadSource
+
+
+def retain_zh_ch_dig(text):
+    return re.sub("[^\u4e00-\u9fa5a-zA-Z0-9\[\]]+", "", text)
+
+
+class RSSSourceFormat:
+    def __init__(self, source):
+        self.source = source
+        self.source["sourceComment"] = ""
+        self.source["sourceUrl"] = self.source["sourceUrl"].rstrip("/|#")
+        if "httpUserAgent" in self.source.keys():
+            self.source["header"] = self.source.pop("httpUserAgent")
+
+        for key in ["sourceGroup", "sourceName"]:
+            self.source[key] = retain_zh_ch_dig(self.source.get(key, ""))
+
+    def run(self):
+        keys = [key for key in self.source.keys() if not self.source[key]]
+        for key in keys:
+            
+            self.source.pop(key)
+
+        for key in ['customOrder', 'respondTime', 'lastUpdateTime']:
+            if key in self.source.keys():
+                self.source.pop(key)
+
+        for key in ['searchUrl', 'exploreUrl']:
+            if key in self.source.keys():
+                self.source[key] = self.source[key].replace(self.source['bookSourceUrl'], '')
+        return self.source
+
+    def __format_base(self, group, map):
+        book_info = self.source.get(group, {})
+
+        for key, name in map.items():
+            if key not in self.source:
+                continue
+            value = self.source.pop(key)
+            if value:
+                book_info[name] = value
+        if len(book_info) > 0:
+            self.source[group] = book_info
+
+
+class RSSSourceDownload(DownloadSource):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def source_format(self, source):
+        return RSSSourceFormat(source).run()
+
+    

funread-1.0.1/src/funread/legado/manage/utils/__init__.py

@@ -0,0 +1 @@
+from .core import url_to_hostname

funread-1.0.1/src/funread/legado/manage/utils/core.py

@@ -0,0 +1,4 @@
+from urllib.parse import urlparse
+
+def url_to_hostname(url):
+    return urlparse(url).hostname

funread-1.0.1/src/funread/scripts/command.py

@@ -0,0 +1,37 @@
+# import argparse
+
+# from funread.legado.task import GenerateSourceTask
+# from funread.legado.task import ReadODSProgressDataTask
+# from funread.legado.task import ReadODSSourceDataTask
+# from funread.legado.task import ReadODSUrlDataTask
+# from funread.legado.task import UpdateRssTask
+
+
+# def funbuild():
+#     parser = argparse.ArgumentParser(prog="PROG")
+#     subparsers = parser.add_subparsers(help="sub-command help")
+
+#     # 添加子命令
+#     build_parser = subparsers.add_parser("url", help="format the source url")
+#     build_parser.set_defaults(func=ReadODSUrlDataTask().run)
+
+#     # 添加子命令
+#     clean_history_parser = subparsers.add_parser("source", help="load the source")
+#     clean_history_parser.set_defaults(func=ReadODSSourceDataTask().run)
+
+#     # 添加子命令
+#     pull_parser = subparsers.add_parser("progress", help="progress the source data.")
+#     pull_parser.add_argument("--quiet", default=True, help="quiet")
+#     pull_parser.set_defaults(func=ReadODSProgressDataTask().run)
+
+#     # 添加子命令
+#     push_parser = subparsers.add_parser("generate", help="git push")
+#     push_parser.add_argument("--quiet", default=True, help="quiet")
+#     push_parser.set_defaults(func=GenerateSourceTask().run)
+
+#     # 添加子命令
+#     tag_parser = subparsers.add_parser("update", help="git build tag")
+#     tag_parser.set_defaults(func=UpdateRssTask().run)
+
+#     args = parser.parse_args()
+#     args.func(args)

funread-1.0.1/src/funread/web/page/video.py

@@ -0,0 +1,12 @@
+from funread.web.parse.video import VideoInfo
+
+from nicegui import ui
+
+
+def video_page(video_info: VideoInfo = None):
+    @ui.page("/media/video/")
+    def video_play():
+        v = ui.video(src=video_info.video_url)
+        v.on("ended", lambda _: ui.notify("Video playback completed"))
+
+    return video_play

funread-1.0.1/src/funread/web/page/video_list.py

@@ -0,0 +1,67 @@
+from funread.web.parse.video import ParseVideo
+
+from nicegui import ui
+from nicegui.events import ValueChangeEventArguments
+from .video import video_page, VideoInfo
+
+
+def show(event: ValueChangeEventArguments):
+    name = type(event.sender).__name__
+    ui.notify(f"{name}: {event.value}")
+
+
+def card(video: VideoInfo):
+    with ui.card():
+        ui.image(video.pic_url)
+        with ui.row():
+            ui.link(video.text, video_page(video))
+
+
+def video_list_page(parse: ParseVideo, tab_id=0):
+    @ui.page("/videos")
+    def video_list_play(rows=10, cols=2):
+        ui.button("Button", on_click=lambda: ui.notify("Click"))
+        videos = parse.parse_video_list()
+
+        index = 0
+        for i in range(rows):
+            with ui.row():
+                for j in range(cols):
+                    if index >= len(videos.video_list):
+                        break
+                    card(videos.video_list[index])
+
+    return video_list_play
+
+
+def video_list_play(parse: ParseVideo, tab_id=0, rows=10, cols=2):
+    ui.button("Button", on_click=lambda: ui.notify("Click"))
+    videos = parse.parse_video_list()
+    print(videos)
+    index = 0
+    for i in range(rows):
+        with ui.row():
+            for j in range(cols):
+                if index >= len(videos.video_list):
+                    break
+                card(videos.video_list[index])
+
+
+def video_list_tabs_page(parse: ParseVideo, tab_id=0):
+    @ui.page("/videosssss")
+    def video_list_tabs_play(rows=10, cols=2):
+        parse_tabs = parse.tabs()
+        if len(parse_tabs) == 0:
+            video_list_play(parse, tab_id=tab_id)
+        else:
+            with ui.tabs().classes("w-full") as tabs:
+                for name in parse_tabs.keys():
+                    tab = ui.tab(name)
+                    with ui.tab_panels(tabs, value=tab).classes("w-full"):
+                        with ui.tab_panel(tab):
+                            video_list_play(parse, tab_id=tab_id)
+
+    return video_list_tabs_play
+
+
+ui.run()

funread-1.0.1/src/funread/web/parse/base.py

@@ -0,0 +1,9 @@
+class BaseParse:
+    def __init__(self, source=None, *args, **kwargs):
+        self.source = source
+
+    def parse_list(self, page_no, page_size=10, *args, **kwargs):
+        pass
+
+    def parse_detail(self, path):
+        pass

funread-1.0.1/src/funread/web/parse/video.py

@@ -0,0 +1,29 @@
+from typing import List
+
+from pydantic import BaseModel
+
+from .base import BaseParse
+
+
+class VideoInfo(BaseModel):
+    text: str = ""
+    pic_url: str = ""
+    video_url: str = ""
+    description: str = None
+
+
+class VideoListInfo(BaseModel):
+    page_no: int = 1
+    page_size = 10
+    video_list: List[VideoInfo] = []
+
+
+class ParseVideo(BaseParse):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def parse_video_detail(self) -> VideoInfo:
+        pass
+
+    def parse_video_list(self) -> VideoListInfo:
+        pass