linkarchivetools 0.1.7__tar.gz → 0.1.10__tar.gz

{linkarchivetools-0.1.7 → linkarchivetools-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: linkarchivetools
-Version: 0.1.7
+Version: 0.1.10
 Summary: Link Archive Tools
 License: GPL3
 Author: Iwan Grozny
@@ -11,7 +11,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: psycopg2-binary
 Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
 Requires-Dist: requests (>=2.32.5,<3.0.0)

{linkarchivetools-0.1.7 → linkarchivetools-0.1.10}/linkarchivetools/utils/reflected.py

@@ -17,19 +17,12 @@ from sqlalchemy import (
 class ReflectedTable(object):
     def __init__(self, engine, connection):
         self.engine = engine
-        self.connection = connection
-
-    def get_table(self, table_name):
-        destination_metadata = MetaData()
-        destination_table = Table(
-            table_name, destination_metadata, autoload_with=self.engine
-        )
-        return destination_table
 
     def truncate_table(self, table_name):
         sql_text = f"DELETE FROM {table_name};"
-        self.connection.execute(text(sql_text))
-        self.connection.commit()
+
+        with self.engine.begin() as connection:
+            connection.execute(text(sql_text))
 
     def create_index(self, table, column_name):
         index_name = f"idx_{table.name}_{column_name}"
@@ -38,30 +31,17 @@ class ReflectedTable(object):
         index.create(bind=self.engine)
 
     def vacuum(self):
-        self.connection.execute(text("VACUUM"))
+        with self.engine.connect() as connection:
+            connection.execution_options(isolation_level="AUTOCOMMIT")
+            connection.execute(text("VACUUM"))
 
     def close(self):
         pass
 
-    def insert_json_data(self, table_name, json_data: dict):
-        table = self.get_table(table_name)
-
-        stmt = (
-            insert(table)
-            .values(**json_data)
-            .returning(table.c.id)
-        )
-
-        result = self.connection.execute(stmt)
-        inserted_id = result.scalar_one()
-        self.connection.commit()
-
-        return inserted_id
-
     def count(self, table_name):
-        row_count = self.connection.execute(
-            text(f"SELECT COUNT(*) FROM {table_name}")
-        ).scalar()
+        sql_text = text(f"SELECT COUNT(*) FROM {table_name}")
+        with self.engine.connect() as connection:
+            row_count = connection.execute(sql_text).scalar_one()
         return row_count
 
     def print_summary(self, print_columns=False):
@@ -92,32 +72,34 @@ class ReflectedTable(object):
         return data
 
     def run_sql(self, sql_text):
-        self.connection.execute(text(sql_text))
-        self.connection.commit()
+        with self.engine.begin() as connection:
+            connection.execute(text(sql_text))
 
 
 class ReflectedGenericTable(object):
     def __init__(self, engine, connection, table_name=None):
         self.engine = engine
-        self.connection = connection
         self.table_name = table_name
         if self.table_name is None:
             self.table_name = self.get_table_name()
+        self.table = None
 
     def get_table_name():
         return self.table_name
 
     def get_table(self):
-        destination_metadata = MetaData()
-        destination_table = Table(
-            self.table_name, destination_metadata, autoload_with=self.engine
-        )
-        return destination_table
+        if self.table is None:
+            destination_metadata = MetaData()
+            self.table = Table(
+                self.table_name, destination_metadata, autoload_with=self.engine
+            )
+            return self.table
+        return self.table
 
     def truncate(self):
         sql_text = f"DELETE FROM {self.table_name};"
-        self.connection.execute(text(sql_text))
-        self.connection.commit()
+        with self.engine.begin() as connection:
+            result = connection.execute(text(sql_text))
 
     def create_index(self, column_name):
         index_name = f"idx_{self.table.name}_{column_name}"
@@ -134,9 +116,9 @@ class ReflectedGenericTable(object):
             .returning(table.c.id)
         )
 
-        result = self.connection.execute(stmt)
-        inserted_id = result.scalar_one()
-        self.connection.commit()
+        with self.engine.begin() as connection:
+            result = connection.execute(stmt)
+            inserted_id = result.scalar_one()
 
         return inserted_id
 
@@ -149,21 +131,22 @@ class ReflectedGenericTable(object):
             .values(**json_data)
         )
 
-        self.connection.execute(stmt)
+        with self.engine.begin() as connection:
+            connection.execute(stmt)
 
     def count(self):
-        row_count = self.connection.execute(
-            text(f"SELECT COUNT(*) FROM {self.table_name}")
-        ).scalar()
+        sql = text(f"SELECT COUNT(*) FROM {self.table_name}")
+        with self.engine.connect() as connection:
+            row_count = connection.execute(sql).scalar_one()
         return row_count
 
     def get(self, id):
-        destination_table = self.get_table()
-
-        stmt = select(destination_table).where(destination_table.c.id == id)
+        table = self.get_table()
+        stmt = select(table).where(table.c.id == id)
 
-        result = self.connection.execute(stmt)
-        return result.first()
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            return result.first()
 
     def get_where(self,
                   conditions_map: dict=None,
@@ -199,36 +182,38 @@ class ReflectedGenericTable(object):
         if limit is not None:
             stmt = stmt.limit(limit)
 
-        result = self.connection.execute(stmt)
-        for row in result:
-            yield row
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            rows = result.fetchall()  # fetch all rows immediately
 
-    def delete(self, id):
-        destination_table = self.get_table()
+        return rows
 
-        stmt = delete(destination_table).where(destination_table.c.id == id)
+    def delete(self, id):
+        table = self.get_table()
+        stmt = delete(table).where(table.c.id == id)
 
-        result = self.connection.execute(stmt)
-        self.connection.commit()
+        with self.engine.begin() as connection:
+            result = connection.execute(stmt)
+            rowcount = result.rowcount  # number of rows deleted
 
-        return result.rowcount  # number of rows deleted
+        return rowcount
 
     def delete_where(self, conditions: dict):
-        destination_table = self.get_table()
+        table = self.get_table()
 
         filters = []
         for column_name, value in conditions.items():
-            if not hasattr(destination_table.c, column_name):
+            if not hasattr(table.c, column_name):
                 raise ValueError(f"Unknown column: {column_name}")
+            filters.append(getattr(table.c, column_name) == value)
 
-            filters.append(getattr(destination_table.c, column_name) == value)
-
-        stmt = delete(destination_table).where(and_(*filters))
+        stmt = delete(table).where(and_(*filters))
 
-        result = self.connection.execute(stmt)
-        self.connection.commit()
+        with self.engine.begin() as connection:
+            result = connection.execute(stmt)
+            rowcount = result.rowcount  # number of rows deleted
 
-        return result.rowcount
+        return rowcount
 
     def print_summary(self, print_columns=False):
         row_count = self.count()
@@ -240,21 +225,23 @@ class ReflectedGenericTable(object):
 
     def get_column_names(self):
         inspector = inspect(self.engine)
-        row_count = self.connection.execute(
-            text(f"SELECT COUNT(*) FROM {self.table_name}")
-        ).scalar()
+
+        with self.engine.connect() as connection:
+            row_count = connection.execute(text(f"SELECT COUNT(*) FROM {self.table_name}")).scalar_one()
 
         columns = inspector.get_columns(self.table_name)
         column_names = [column["name"] for column in columns]
         return column_names
 
     def row_to_json_data(self, row):
-        data = dict(row._mapping)
-        return data
+        """
+        Convert SQLAlchemy row to a dict
+        """
+        return dict(row._mapping)
 
     def run_sql(self, sql_text):
-        self.connection.execute(text(sql_text))
-        self.connection.commit()
+        with self.engine.begin() as connection:
+            connection.execute(text(sql_text))
 
 
 class ReflectedEntryTable(ReflectedGenericTable):
@@ -286,35 +273,38 @@ class ReflectedEntryTable(ReflectedGenericTable):
 
         return self.insert_json_data(entry_json)
 
-    def get_entries(self, limit:int|None=None, offset:int=0):
-        destination_table = self.get_table()
-
-        entries_select = select(destination_table)
+    def get_entries(self, limit: int | None = None, offset: int = 0):
+        """
+        TODO remove use get_where
+        """
+        table = self.get_table()
+        stmt = select(table)
 
         if offset:
-            entries_select = entries_select.offset(offset)
+            stmt = stmt.offset(offset)
         if limit is not None:
-            entries_select = entries_select.limit(limit)
-
-        result = self.connection.execute(entries_select)
+            stmt = stmt.limit(limit)
 
-        for entry in result:
-            yield entry
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            rows = result.fetchall()  # fetch all rows immediately
+        return rows
 
     def get_entries_good(self):
-        destination_table = self.get_table()
-
+        """
+        TODO remove use get_where
+        """
+        table = self.get_table()
         stmt = (
-            select(destination_table)
-            .where(destination_table.c.page_rating_votes > 0)
-            .order_by(destination_table.c.page_rating_votes.desc())
+            select(table)
+            .where(table.c.page_rating_votes > 0)
+            .order_by(table.c.page_rating_votes.desc())
         )
 
-        result = self.connection.execute(stmt)
-        entries = result.fetchall()
-
-        for entry in entries:
-            yield entry
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            rows = result.fetchall()  # fetch all rows immediately
+        return rows
 
     def exists(self, *, id=None, link=None):
         table = self.get_table()
@@ -329,7 +319,9 @@ class ReflectedEntryTable(ReflectedGenericTable):
             return False
 
         stmt = select(exists().where(or_(*conditions)))
-        return self.connection.execute(stmt).scalar()
+
+        with self.engine.connect() as connection:
+            return connection.execute(stmt).scalar()
 
 
 class ReflectedUserTags(ReflectedGenericTable):
@@ -337,33 +329,27 @@ class ReflectedUserTags(ReflectedGenericTable):
         return "usertags"
 
     def get_tags_string(self, entry_id):
-        destination_table = self.get_table()
-
-        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
+        table = self.get_table()
+        stmt = select(table).where(table.c.entry_id == entry_id)
 
-        tags = ""
+        tags_list = []
 
-        result = self.connection.execute(stmt)
-        rows = result.fetchall()
-        for row in rows:
-            if tags:
-                tags += ", "
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            for row in result:
+                tags_list.append(f"#{row.tag}")
 
-            tags += "#" + row.tag
-
-        return tags
+        return ", ".join(tags_list)
 
     def get_tags(self, entry_id):
-        destination_table = self.get_table()
-
-        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
+        table = self.get_table()
+        stmt = select(table).where(table.c.entry_id == entry_id)
 
         tags = []
-
-        result = self.connection.execute(stmt)
-        rows = result.fetchall()
-        for row in rows:
-            tags.append(row.tag)
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            for row in result:
+                tags.append(row.tag)
 
         return tags
 
@@ -372,36 +358,32 @@ class ReflectedEntryCompactedTags(ReflectedGenericTable):
     def get_table_name(self):
         return "entrycompactedtags"
 
-    def get_tags_string(self, entry_id):
-        destination_table = self.get_table()
-
-        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
-
-        tags = ""
-
-        result = self.connection.execute(stmt)
-        rows = result.fetchall()
-        for row in rows:
-            if tags:
-                tags += ", "
+    def get_tags(self, entry_id):
+        """Return a list of tag strings for the given entry_id."""
+        table = self.get_table()
+        stmt = select(table).where(table.c.entry_id == entry_id)
 
-            tags += "#" + row.tag
+        tags = []
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            for row in result:
+                tags.append(row.tag)
 
         return tags
 
-    def get_tags(self, entry_id):
-        destination_table = self.get_table()
 
-        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
-
-        tags = []
+    def get_tags_string(self, entry_id):
+        """Return tags for the given entry_id as a single string formatted as '#tag1, #tag2'."""
+        table = self.get_table()
+        stmt = select(table).where(table.c.entry_id == entry_id)
 
-        result = self.connection.execute(stmt)
-        rows = result.fetchall()
-        for row in rows:
-            tags.append(row.tag)
+        tags_list = []
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            for row in result:
+                tags_list.append(f"#{row.tag}")
 
-        return tags
+        return ", ".join(tags_list)
 
 
 class ReflectedSourceTable(ReflectedGenericTable):
@@ -409,35 +391,35 @@ class ReflectedSourceTable(ReflectedGenericTable):
         return "sourcedatamodel"
 
     def get_source(self, source_id):
-        destination_table = self.get_table()
-
-        stmt = select(destination_table).where(destination_table.c.id == source_id)
-
-        result = self.connection.execute(stmt)
-        return result.first()
+        """Return a single source row by ID, or None if not found."""
+        table = self.get_table()
+        stmt = select(table).where(table.c.id == source_id)
 
-    def get_sources(self, limit:int|None=None, offset:int=0):
-        destination_table = self.get_table()
+        with self.engine.connect() as connection:
+            return connection.execute(stmt).first()
 
-        sources_select = select(destination_table)
+    def get_sources(self, limit: int | None = None, offset: int = 0):
+        """Yield sources with optional offset and limit."""
+        table = self.get_table()
+        stmt = select(table)
 
         if offset:
-            sources_select = sources_select.offset(offset)
+            stmt = stmt.offset(offset)
         if limit is not None:
-            sources_select = sources_select.limit(limit)
-
-        result = self.connection.execute(sources_select)
-
-        for source in result:
-            yield source
+            stmt = stmt.limit(limit)
 
-    def insert_json(self, source_json):
-        if "url" not in source_json:
-            source_json["url"] = ""
+        with self.engine.connect() as connection:
+            result = connection.execute(stmt)
+            sources = result.fetchall()
+        return sources
 
+    def insert_json(self, source_json: dict):
+        """Insert a source JSON dict, ensuring 'url' key exists."""
+        source_json.setdefault("url", "")
         return self.insert_json_data(source_json)
 
     def exists(self, *, id=None, url=None):
+        """Return True if a source with given ID or URL exists."""
         table = self.get_table()
 
         conditions = []
@@ -450,7 +432,9 @@ class ReflectedSourceTable(ReflectedGenericTable):
             return False
 
         stmt = select(exists().where(or_(*conditions)))
-        return self.connection.execute(stmt).scalar()
+
+        with self.engine.connect() as connection:
+            return connection.execute(stmt).scalar()
 
 
 class ReflectedSocialData(ReflectedGenericTable):
@@ -458,20 +442,20 @@ class ReflectedSocialData(ReflectedGenericTable):
         return "socialdata"
 
     def get(self, entry_id):
-        destination_table = self.get_table()
+        """Return a single row matching entry_id, or None if not found."""
+        table = self.get_table()
+        stmt = select(table).where(table.c.entry_id == entry_id)
 
-        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
+        with self.engine.connect() as connection:
+            return connection.execute(stmt).first()
 
-        result = self.connection.execute(stmt)
-        return result.first()
 
     def get_json(self, entry_id):
+        """Return the row as a dict (JSON-style), or None if not found."""
         row = self.get(entry_id)
         if row is None:
             return None
-
-        data = self.row_to_json_data(row)
-        return data
+        return self.row_to_json_data(row)
 
 
 class EntryCopier(object):

{linkarchivetools-0.1.7 → linkarchivetools-0.1.10}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "linkarchivetools"
-version = "0.1.7"
+version = "0.1.10"
 description = "Link Archive Tools"
 authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
 license = "GPL3"