linkarchivetools 0.1.10__py3-none-any.whl

linkarchivetools/db2feeds.py

@@ -0,0 +1,263 @@
+"""
+Converts Database to information about RSS feeds.
+
+TODO - copy tags from origin to RSS feeds
+"""
+
+import shutil
+import argparse
+from pathlib import Path
+from sqlalchemy import create_engine
+
+from webtoolkit import RemoteUrl, BaseUrl
+from linkarchivetools import tableconfig
+from .utils.reflected import *
+
+
+class Db2Feeds(object):
+    """
+    Converter DB -> feeds.
+    """
+
+    def __init__(
+        self,
+        input_db=None,
+        output_db=None,
+        verbose=True,
+        clean=False,
+        remote_server="",
+        output_format=None,
+        read_internet_links=False,
+        update_feed=False,
+    ):
+        """
+        Constructor
+        @param read_internet_links Read links to find RSS feeds
+        @param update_feed Many things are copied from original entry.
+                          If this setting is true, feed entry fetches title, and other properties
+        """
+        self.input_db = input_db
+        self.output_db = output_db
+        self.verbose = verbose
+        self.clean = clean
+        self.remote_server = remote_server
+        self.output_format = output_format
+        self.read_internet_links = read_internet_links
+        self.update_feed = update_feed
+
+        self.new_table = None
+
+        if self.output_db:
+            self.output_format = "SQLITE"
+        self.make_output_db()
+
+    def make_output_db(self):
+        if self.output_format != "SQLITE":
+            self.new_engine = None
+            self.new_connection = None
+            return
+
+        new_path = Path(self.output_db)
+        if new_path.exists() and self.clean:
+            new_path.unlink()
+
+        if not new_path.exists():
+            shutil.copy(self.input_db, self.output_db)
+            self.new_engine = create_engine(f"sqlite:///{self.output_db}")
+            with self.new_engine.connect() as new_connection:
+                self.new_connection = new_connection
+                self.truncate_tables()
+
+        self.new_engine = create_engine(f"sqlite:///{self.output_db}")
+
+    def convert(self):
+        """
+        API
+        """
+        self.engine = create_engine(f"sqlite:///{self.input_db}")
+        with self.engine.connect() as connection:
+            self.connection = connection
+            if self.new_engine:
+                with self.new_engine.connect() as new_connection:
+                    self.new_connection = new_connection
+                    self.convert_entries()
+            else:
+                self.convert_entries()
+
+    def convert_entries(self):
+        if self.clean:
+            self.truncate_tables()
+
+        self.new_table = None
+        if self.new_engine:
+            self.new_table = ReflectedEntryTable(self.new_engine, self.new_connection)
+
+        table = ReflectedEntryTable(self.engine, self.connection)
+        for entry in table.get_entries_good():
+            self.convert_entry(entry)
+
+    def convert_entry(self, entry):
+        url = BaseUrl(entry.link)
+        feeds = url.get_feeds()
+
+        if len(feeds) == 0:
+            if self.read_internet_links:
+                if self.remote_server:
+                    url_ex = RemoteUrl(remote_server_location=self.remote_server, url=entry.link)
+                    url_ex.get_response()
+                    feeds.extend(url_ex.get_feeds())
+
+        for feed in feeds:
+            data = self.prepare_data(entry, feed)
+
+            if self.new_table:
+                if not self.new_table.exists(link=feed):
+                    self.print_data(entry, data)
+                    self.copy_entry(entry, self.new_table, data)
+            else:
+                self.print_data(entry, data)
+
+    def prepare_data(self, entry, feed):
+        data = {}
+        data["link"] = feed
+        data["title"] = entry.title
+        data["page_rating_votes"] = entry.page_rating_votes
+        data["manual_status_code"] = entry.manual_status_code
+        data["thumbnail"] = entry.thumbnail
+        data["language"] = entry.language
+
+        # not null requirement
+        data["source_url"] = ""
+        data["permanent"] = False
+        data["bookmarked"] = False
+        data["status_code"] = entry.status_code
+        data["contents_type"] = 0
+        data["page_rating_contents"] = 0
+        data["page_rating_visits"] = 0
+        data["page_rating"] = 0
+
+        if self.update_feed and self.remote_server:
+            url_feed = RemoteUrl(remote_server=self.remote_server, url=feed)
+            url_feed.get_response()
+
+            data["title"] = url_feed.get_title()
+            data["description"] = url_feed.get_description()
+            data["status_code"] = url_feed.get_status_code()
+            data["thumbnail"] = url_feed.get_thumbnail()
+        return data
+
+    def copy_entry(self, entry, entry_table, data):
+        """
+        """
+        new_entry_id = entry_table.insert_json(data)
+        self.copy_tags(entry, new_entry_id)
+        self.copy_social_data(entry, new_entry_id)
+
+    def copy_tags(self, entry, new_entry_id):
+        source_entry_compacted_tags = ReflectedEntryCompactedTags(self.engine, self.connection)
+        tags = source_entry_compacted_tags.get_tags(entry.id)
+
+        entry_tag_data = {}
+        for tag in tags:
+            entry_tag_data["tag"] = tag
+            entry_tag_data["entry_id"] = new_entry_id
+            destination_entry_compacted_tags = ReflectedEntryCompactedTags(self.new_engine, self.new_connection)
+            destination_entry_compacted_tags.insert_json_data(entry_tag_data)
+
+    def copy_social_data(self, entry, new_entry_id):
+        source_entry_social_data = ReflectedSocialData(self.engine, self.connection)
+        social_data = source_entry_social_data.get_json(entry.id)
+        if social_data:
+            if "id" in social_data:
+                del social_data["id"]
+            social_data["entry_id"] = new_entry_id
+
+            destination_entry_social_data = ReflectedSocialData(self.new_engine, self.new_connection)
+            destination_entry_social_data.insert_json_data(social_data)
+
+    def truncate_tables(self):
+        if not self.new_engine:
+            return
+
+        table_names = self.get_table_names()
+        for table_name in table_names:
+            table = ReflectedTable(self.new_engine, self.new_connection)
+            table.truncate_table(table_name)
+
+        table = ReflectedTable(self.new_engine, self.new_connection)
+        table.vacuum()
+
+    def get_table_names(self):
+        return tableconfig.get_backup_tables()
+
+    def print_data(self, entry, data):
+        """
+        If we print to SQLITE we want to see progress so we display it anyway
+        """
+        if not self.verbose:
+            return
+
+        link = data["link"]
+        title = data["title"]
+        page_rating_votes = data["page_rating_votes"]
+
+        if self.output_format == "LINES" or self.output_format == "SQLITE":
+            print(f"[{page_rating_votes}] {link} - {title}")
+            user_tags = ReflectedEntryCompactedTags(self.engine, self.connection)
+            tags = user_tags.get_tags_string(entry.id)
+            if tags:
+                print(f"{tags}")
+        elif self.output_format == "JSON":
+            user_tags = ReflectedEntryCompactedTags(self.engine, self.connection)
+            tags = user_tags.get_tags(entry.id)
+            print(
+                f"""
+            \{ "title" : "{title}",
+              "link" : "{link}",
+              "page_rating_votes : {page_rating_votes},
+              "tags" : {tags}
+            \}"""
+            )
+        else:
+            print("Unsupported output format")
+
+
+def parse():
+    parser = argparse.ArgumentParser(description="Data analyzer program")
+    parser.add_argument("--db", default="catalog.db", help="DB to be scanned")
+    parser.add_argument("--output-db", help="File to be created")
+    parser.add_argument("--update-rss",action="store_true", help="Reads RSS to check it's title and properties")
+    parser.add_argument("--clean",action="store_true", help="If output db exists, then it is removed at start")
+    parser.add_argument("--read-internet-links",action="store_true", help="Reads entries to check if contains RSS. Without it only calculated RSS are returned")
+    parser.add_argument(
+        "--output-format",
+        default="LINES",
+        help="format of display. LINES, JSON, SQLITE",
+    )
+    parser.add_argument("--crawling-server", default="", help="Remote crawling server")
+
+    args = parser.parse_args()
+
+    return parser, args
+
+
+def main():
+    parser, args = parse()
+
+    path = Path(args.db)
+    if not path.exists():
+        print("File {} does not exist".format(path))
+        return
+
+    reader = Db2Feeds(
+        input_db=args.db,
+        output_db=args.output_db,
+        clean=args.clean,
+        remote_server=args.crawling_server,
+        output_format=args.output_format,
+    )
+    reader.convert()
+
+
+if __name__ == "__main__":
+    main()

linkarchivetools/db2json.py

@@ -0,0 +1,188 @@
+"""
+Converts database to JSON.
+"""
+
+import os
+import sys
+import json
+import shutil
+from pathlib import Path
+import argparse
+
+from sqlalchemy import create_engine
+from .utils.reflected import *
+
+
+class Db2JSON(object):
+
+    def __init__(self, input_db, output_dir, format=None, rows_max=1000):
+        self.input_db = input_db
+        self.output_dir = output_dir
+
+        self.format = format
+        self.rows_max = rows_max
+
+        self.file_index = 0
+        self.entry_index = 0
+        self.handle = None
+
+        self.rows = []
+
+        self.processed = 0
+        self.all = 0
+
+        self.setup()
+
+    def setup(self):
+        path = Path(self.input_db)
+        if not path.exists():
+            print("File {} does not exist".format(path))
+            return
+
+        if self.output_dir and self.output_dir != ".":
+            new_path = Path(self.output_dir)
+            if new_path.exists():
+                shutil.rmtree(new_path)
+            new_path.mkdir()
+
+        self.engine = create_engine(f"sqlite:///{self.input_db}")
+
+    def write(self, entry):
+        """Write entries to the specified directory, 1000 per file."""
+        if self.handle == None:
+            file_path = str(self.get_file_path())
+            self.handle = open(file_path, "w")
+
+        row = self.get_entry_json_data(entry)
+
+        self.rows.append(row)
+
+        self.entry_index += 1
+
+        sys.stdout.write(f"{self.file_index}/{self.entry_index:04d}\r")
+
+        if self.entry_index == self.rows_max:
+            self.file_index += 1
+            self.entry_index = 0
+            self.finish_stream()
+
+            file_path = str(self.get_file_path())
+            self.handle = open(file_path, "w")
+
+    def get_entry_json_data(self, entry):
+        date_published = entry.date_published
+        if date_published:
+            date_published = date_published.isoformat()
+
+        date_dead_since = entry.date_dead_since
+        if date_dead_since:
+            date_dead_since = date_dead_since.isoformat()
+
+        row = {
+            "link": entry.link,
+            "description": entry.description,
+            "author": entry.author,
+            "album": entry.album,
+            "bookmarked": entry.bookmarked,
+            "date_dead_since": date_dead_since,
+            "date_published": date_published,
+            "language": entry.language,
+            "manual_status_code": entry.manual_status_code,
+            "page_rating": entry.page_rating,
+            "page_rating_contents": entry.page_rating_contents,
+            "page_rating_votes": entry.page_rating_votes,
+            "page_rating_visits": entry.page_rating_visits,
+            "permanent": entry.permanent,
+            "source_url": entry.source_url,
+            "status_code": entry.status_code,
+            "thumbnail": entry.thumbnail,
+            "title": entry.title,
+            "age": entry.age,
+            "id": entry.id,
+        }
+
+        social_table = ReflectedSocialData(self.engine, self.connection)
+        social_data = social_table.get(entry.id)
+        if social_data:
+            row.setdefault("thumbs_up", social_data.thumbs_up)
+            row.setdefault("thumbs_down", social_data.thumbs_down)
+            row.setdefault("view_count", social_data.view_count)
+            row.setdefault("rating", social_data.rating)
+            row.setdefault("upvote_ratio", social_data.upvote_ratio)
+            row.setdefault("upvote_diff", social_data.upvote_diff)
+            row.setdefault("upvote_view_ratio", social_data.upvote_view_ratio)
+            row.setdefault("stars", social_data.stars)
+            row.setdefault("followers_count", social_data.followers_count)
+
+        tags_table = ReflectedUserTags(self.engine, self.connection)
+        tags = tags_table.get_tags(entry.id)
+        row["tags"] = tags
+
+        return row
+
+    def get_file_path(self):
+        filename = "{}_{}.json".format(self.format, str(self.file_index))
+        if self.output_dir and self.output_dir != ".":
+            return Path(self.output_dir) / filename
+        else:
+            return Path(filename)
+
+    def close(self):
+        if self.handle:
+            self.finish_stream()
+        self.handle = None
+
+    def finish_stream(self):
+        if not self.handle:
+            return
+
+        try:
+            string = json.dumps(self.rows, indent=4)
+            self.handle.write(string)
+        except ValueError as e:
+            print(f"Error writing file {file_path}: {e}")
+        self.handle.close()
+        self.handle = None
+        self.rows = []
+
+    def convert(self):
+        with self.engine.connect() as connection:
+            self.connection = connection
+            table = ReflectedEntryTable(self.engine, connection)
+
+            for entry in table.get_entries():
+                # print(entry)
+                self.write(entry)
+
+        self.close()
+
+
+def parse():
+    parser = argparse.ArgumentParser(description="Data analyzer program")
+    parser.add_argument("--db", default="places.db", help="DB to be scanned")
+    parser.add_argument("--output-dir", default="json", help="Output directory")
+    parser.add_argument(
+        "--rows-max", default=1000, action="store_true", help="Number of rows per file"
+    )
+    parser.add_argument("-f", "--format", default="entries", help="file name format")
+    parser.add_argument("-v", "--verbosity", help="Verbosity level")
+
+    args = parser.parse_args()
+
+    return parser, args
+
+
+def main():
+    parser, args = parse()
+
+    f = Db2JSON(
+        input_db=args.db,
+        output_dir=args.output_dir,
+        format=args.format,
+        rows_max=args.rows_max,
+    )
+    f.convert()
+
+
+if __name__ == "__main__":
+    main()