linkarchivetools 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

linkarchivetools/utils/reflected.py

@@ -1,3 +1,8 @@
+"""
+This file is mainly for SQLite.
+It will not open several connection, will use one.
+This allows us to handle nested calls of generators without any problems.
+"""
 from sqlalchemy import (
     MetaData,
     Table,
@@ -17,12 +22,19 @@ from sqlalchemy import (
 class ReflectedTable(object):
     def __init__(self, engine, connection):
         self.engine = engine
+        self.connection = connection
+
+    def get_table(self, table_name):
+        destination_metadata = MetaData()
+        destination_table = Table(
+            table_name, destination_metadata, autoload_with=self.engine
+        )
+        return destination_table
 
     def truncate_table(self, table_name):
         sql_text = f"DELETE FROM {table_name};"
-
-        with self.engine.begin() as connection:
-            connection.execute(text(sql_text))
+        self.connection.execute(text(sql_text))
+        self.connection.commit()
 
     def create_index(self, table, column_name):
         index_name = f"idx_{table.name}_{column_name}"
@@ -31,17 +43,30 @@ class ReflectedTable(object):
         index.create(bind=self.engine)
 
     def vacuum(self):
-        with self.engine.connect() as connection:
-            connection.execution_options(isolation_level="AUTOCOMMIT")
-            connection.execute(text("VACUUM"))
+        self.connection.execute(text("VACUUM"))
 
     def close(self):
         pass
 
+    def insert_json_data(self, table_name, json_data: dict):
+        table = self.get_table(table_name)
+
+        stmt = (
+            insert(table)
+            .values(**json_data)
+            .returning(table.c.id)
+        )
+
+        result = self.connection.execute(stmt)
+        inserted_id = result.scalar_one()
+        self.connection.commit()
+
+        return inserted_id
+
     def count(self, table_name):
-        sql_text = text(f"SELECT COUNT(*) FROM {table_name}")
-        with self.engine.connect() as connection:
-            row_count = connection.execute(sql_text).scalar_one()
+        row_count = self.connection.execute(
+            text(f"SELECT COUNT(*) FROM {table_name}")
+        ).scalar()
         return row_count
 
     def print_summary(self, print_columns=False):
@@ -72,34 +97,32 @@ class ReflectedTable(object):
         return data
 
     def run_sql(self, sql_text):
-        with self.engine.begin() as connection:
-            connection.execute(text(sql_text))
+        self.connection.execute(text(sql_text))
+        self.connection.commit()
 
 
 class ReflectedGenericTable(object):
     def __init__(self, engine, connection, table_name=None):
         self.engine = engine
+        self.connection = connection
         self.table_name = table_name
         if self.table_name is None:
             self.table_name = self.get_table_name()
-        self.table = None
 
     def get_table_name():
         return self.table_name
 
     def get_table(self):
-        if self.table is None:
-            destination_metadata = MetaData()
-            self.table = Table(
-                self.table_name, destination_metadata, autoload_with=self.engine
-            )
-            return self.table
-        return self.table
+        destination_metadata = MetaData()
+        destination_table = Table(
+            self.table_name, destination_metadata, autoload_with=self.engine
+        )
+        return destination_table
 
     def truncate(self):
         sql_text = f"DELETE FROM {self.table_name};"
-        with self.engine.begin() as connection:
-            result = connection.execute(text(sql_text))
+        self.connection.execute(text(sql_text))
+        self.connection.commit()
 
     def create_index(self, column_name):
         index_name = f"idx_{self.table.name}_{column_name}"
@@ -116,9 +139,9 @@ class ReflectedGenericTable(object):
             .returning(table.c.id)
         )
 
-        with self.engine.begin() as connection:
-            result = connection.execute(stmt)
-            inserted_id = result.scalar_one()
+        result = self.connection.execute(stmt)
+        inserted_id = result.scalar_one()
+        self.connection.commit()
 
         return inserted_id
 
@@ -131,22 +154,21 @@ class ReflectedGenericTable(object):
             .values(**json_data)
         )
 
-        with self.engine.begin() as connection:
-            connection.execute(stmt)
+        self.connection.execute(stmt)
 
     def count(self):
-        sql = text(f"SELECT COUNT(*) FROM {self.table_name}")
-        with self.engine.connect() as connection:
-            row_count = connection.execute(sql).scalar_one()
+        row_count = self.connection.execute(
+            text(f"SELECT COUNT(*) FROM {self.table_name}")
+        ).scalar()
         return row_count
 
     def get(self, id):
-        table = self.get_table()
-        stmt = select(table).where(table.c.id == id)
+        destination_table = self.get_table()
 
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            return result.first()
+        stmt = select(destination_table).where(destination_table.c.id == id)
+
+        result = self.connection.execute(stmt)
+        return result.first()
 
     def get_where(self,
                   conditions_map: dict=None,
@@ -182,38 +204,36 @@ class ReflectedGenericTable(object):
         if limit is not None:
             stmt = stmt.limit(limit)
 
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            rows = result.fetchall()  # fetch all rows immediately
-
-        return rows
+        result = self.connection.execute(stmt)
+        for row in result:
+            yield row
 
     def delete(self, id):
-        table = self.get_table()
-        stmt = delete(table).where(table.c.id == id)
+        destination_table = self.get_table()
+
+        stmt = delete(destination_table).where(destination_table.c.id == id)
 
-        with self.engine.begin() as connection:
-            result = connection.execute(stmt)
-            rowcount = result.rowcount  # number of rows deleted
+        result = self.connection.execute(stmt)
+        self.connection.commit()
 
-        return rowcount
+        return result.rowcount  # number of rows deleted
 
     def delete_where(self, conditions: dict):
-        table = self.get_table()
+        destination_table = self.get_table()
 
         filters = []
         for column_name, value in conditions.items():
-            if not hasattr(table.c, column_name):
+            if not hasattr(destination_table.c, column_name):
                 raise ValueError(f"Unknown column: {column_name}")
-            filters.append(getattr(table.c, column_name) == value)
 
-        stmt = delete(table).where(and_(*filters))
+            filters.append(getattr(destination_table.c, column_name) == value)
+
+        stmt = delete(destination_table).where(and_(*filters))
 
-        with self.engine.begin() as connection:
-            result = connection.execute(stmt)
-            rowcount = result.rowcount  # number of rows deleted
+        result = self.connection.execute(stmt)
+        self.connection.commit()
 
-        return rowcount
+        return result.rowcount
 
     def print_summary(self, print_columns=False):
         row_count = self.count()
@@ -225,23 +245,21 @@ class ReflectedGenericTable(object):
 
     def get_column_names(self):
         inspector = inspect(self.engine)
-
-        with self.engine.connect() as connection:
-            row_count = connection.execute(text(f"SELECT COUNT(*) FROM {self.table_name}")).scalar_one()
+        row_count = self.connection.execute(
+            text(f"SELECT COUNT(*) FROM {self.table_name}")
+        ).scalar()
 
         columns = inspector.get_columns(self.table_name)
         column_names = [column["name"] for column in columns]
         return column_names
 
     def row_to_json_data(self, row):
-        """
-        Convert SQLAlchemy row to a dict
-        """
-        return dict(row._mapping)
+        data = dict(row._mapping)
+        return data
 
     def run_sql(self, sql_text):
-        with self.engine.begin() as connection:
-            connection.execute(text(sql_text))
+        self.connection.execute(text(sql_text))
+        self.connection.commit()
 
 
 class ReflectedEntryTable(ReflectedGenericTable):
@@ -273,38 +291,35 @@ class ReflectedEntryTable(ReflectedGenericTable):
 
         return self.insert_json_data(entry_json)
 
-    def get_entries(self, limit: int | None = None, offset: int = 0):
-        """
-        TODO remove use get_where
-        """
-        table = self.get_table()
-        stmt = select(table)
+    def get_entries(self, limit:int|None=None, offset:int=0):
+        destination_table = self.get_table()
+
+        entries_select = select(destination_table)
 
         if offset:
-            stmt = stmt.offset(offset)
+            entries_select = entries_select.offset(offset)
         if limit is not None:
-            stmt = stmt.limit(limit)
+            entries_select = entries_select.limit(limit)
+
+        result = self.connection.execute(entries_select)
 
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            rows = result.fetchall()  # fetch all rows immediately
-        return rows
+        for entry in result:
+            yield entry
 
     def get_entries_good(self):
-        """
-        TODO remove use get_where
-        """
-        table = self.get_table()
+        destination_table = self.get_table()
+
         stmt = (
-            select(table)
-            .where(table.c.page_rating_votes > 0)
-            .order_by(table.c.page_rating_votes.desc())
+            select(destination_table)
+            .where(destination_table.c.page_rating_votes > 0)
+            .order_by(destination_table.c.page_rating_votes.desc())
         )
 
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            rows = result.fetchall()  # fetch all rows immediately
-        return rows
+        result = self.connection.execute(stmt)
+        entries = result.fetchall()
+
+        for entry in entries:
+            yield entry
 
     def exists(self, *, id=None, link=None):
         table = self.get_table()
@@ -319,9 +334,7 @@ class ReflectedEntryTable(ReflectedGenericTable):
             return False
 
         stmt = select(exists().where(or_(*conditions)))
-
-        with self.engine.connect() as connection:
-            return connection.execute(stmt).scalar()
+        return self.connection.execute(stmt).scalar()
 
 
 class ReflectedUserTags(ReflectedGenericTable):
@@ -329,27 +342,33 @@ class ReflectedUserTags(ReflectedGenericTable):
         return "usertags"
 
     def get_tags_string(self, entry_id):
-        table = self.get_table()
-        stmt = select(table).where(table.c.entry_id == entry_id)
+        destination_table = self.get_table()
+
+        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
 
-        tags_list = []
+        tags = ""
 
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            for row in result:
-                tags_list.append(f"#{row.tag}")
+        result = self.connection.execute(stmt)
+        rows = result.fetchall()
+        for row in rows:
+            if tags:
+                tags += ", "
 
-        return ", ".join(tags_list)
+            tags += "#" + row.tag
+
+        return tags
 
     def get_tags(self, entry_id):
-        table = self.get_table()
-        stmt = select(table).where(table.c.entry_id == entry_id)
+        destination_table = self.get_table()
+
+        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
 
         tags = []
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            for row in result:
-                tags.append(row.tag)
+
+        result = self.connection.execute(stmt)
+        rows = result.fetchall()
+        for row in rows:
+            tags.append(row.tag)
 
         return tags
 
@@ -358,32 +377,36 @@ class ReflectedEntryCompactedTags(ReflectedGenericTable):
     def get_table_name(self):
         return "entrycompactedtags"
 
-    def get_tags(self, entry_id):
-        """Return a list of tag strings for the given entry_id."""
-        table = self.get_table()
-        stmt = select(table).where(table.c.entry_id == entry_id)
+    def get_tags_string(self, entry_id):
+        destination_table = self.get_table()
 
-        tags = []
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            for row in result:
-                tags.append(row.tag)
+        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
+
+        tags = ""
+
+        result = self.connection.execute(stmt)
+        rows = result.fetchall()
+        for row in rows:
+            if tags:
+                tags += ", "
+
+            tags += "#" + row.tag
 
         return tags
 
+    def get_tags(self, entry_id):
+        destination_table = self.get_table()
 
-    def get_tags_string(self, entry_id):
-        """Return tags for the given entry_id as a single string formatted as '#tag1, #tag2'."""
-        table = self.get_table()
-        stmt = select(table).where(table.c.entry_id == entry_id)
+        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
 
-        tags_list = []
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            for row in result:
-                tags_list.append(f"#{row.tag}")
+        tags = []
 
-        return ", ".join(tags_list)
+        result = self.connection.execute(stmt)
+        rows = result.fetchall()
+        for row in rows:
+            tags.append(row.tag)
+
+        return tags
 
 
 class ReflectedSourceTable(ReflectedGenericTable):
@@ -391,35 +414,35 @@ class ReflectedSourceTable(ReflectedGenericTable):
         return "sourcedatamodel"
 
     def get_source(self, source_id):
-        """Return a single source row by ID, or None if not found."""
-        table = self.get_table()
-        stmt = select(table).where(table.c.id == source_id)
+        destination_table = self.get_table()
 
-        with self.engine.connect() as connection:
-            return connection.execute(stmt).first()
+        stmt = select(destination_table).where(destination_table.c.id == source_id)
 
-    def get_sources(self, limit: int | None = None, offset: int = 0):
-        """Yield sources with optional offset and limit."""
-        table = self.get_table()
-        stmt = select(table)
+        result = self.connection.execute(stmt)
+        return result.first()
+
+    def get_sources(self, limit:int|None=None, offset:int=0):
+        destination_table = self.get_table()
+
+        sources_select = select(destination_table)
 
         if offset:
-            stmt = stmt.offset(offset)
+            sources_select = sources_select.offset(offset)
         if limit is not None:
-            stmt = stmt.limit(limit)
+            sources_select = sources_select.limit(limit)
+
+        result = self.connection.execute(sources_select)
+
+        for source in result:
+            yield source
 
-        with self.engine.connect() as connection:
-            result = connection.execute(stmt)
-            sources = result.fetchall()
-        return sources
+    def insert_json(self, source_json):
+        if "url" not in source_json:
+            source_json["url"] = ""
 
-    def insert_json(self, source_json: dict):
-        """Insert a source JSON dict, ensuring 'url' key exists."""
-        source_json.setdefault("url", "")
         return self.insert_json_data(source_json)
 
     def exists(self, *, id=None, url=None):
-        """Return True if a source with given ID or URL exists."""
         table = self.get_table()
 
         conditions = []
@@ -432,9 +455,7 @@ class ReflectedSourceTable(ReflectedGenericTable):
             return False
 
         stmt = select(exists().where(or_(*conditions)))
-
-        with self.engine.connect() as connection:
-            return connection.execute(stmt).scalar()
+        return self.connection.execute(stmt).scalar()
 
 
 class ReflectedSocialData(ReflectedGenericTable):
@@ -442,20 +463,20 @@ class ReflectedSocialData(ReflectedGenericTable):
         return "socialdata"
 
     def get(self, entry_id):
-        """Return a single row matching entry_id, or None if not found."""
-        table = self.get_table()
-        stmt = select(table).where(table.c.entry_id == entry_id)
+        destination_table = self.get_table()
 
-        with self.engine.connect() as connection:
-            return connection.execute(stmt).first()
+        stmt = select(destination_table).where(destination_table.c.entry_id == entry_id)
 
+        result = self.connection.execute(stmt)
+        return result.first()
 
     def get_json(self, entry_id):
-        """Return the row as a dict (JSON-style), or None if not found."""
         row = self.get(entry_id)
         if row is None:
             return None
-        return self.row_to_json_data(row)
+
+        data = self.row_to_json_data(row)
+        return data
 
 
 class EntryCopier(object):

{linkarchivetools-0.1.10.dist-info → linkarchivetools-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: linkarchivetools
-Version: 0.1.10
+Version: 0.1.11
 Summary: Link Archive Tools
 License: GPL3
 Author: Iwan Grozny
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: psycopg2-binary
 Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
 Requires-Dist: requests (>=2.32.5,<3.0.0)

{linkarchivetools-0.1.10.dist-info → linkarchivetools-0.1.11.dist-info}/RECORD

@@ -11,8 +11,8 @@ linkarchivetools/json2db.py,sha256=IF01_KhMmHcdm4jfagU_kuJGCEtRwgwjZDuxLtlT7EQ,7
 linkarchivetools/tableconfig.py,sha256=yPhxrU0ioo_uhzr03JTPi8xkPV5sdpCtCgXxyAP8EKE,1585
 linkarchivetools/utils/alchemysearch.py,sha256=voUDEBP4RShlGj8E0iHg83jP8JYJQKLOaJjCZNi4LMY,5689
 linkarchivetools/utils/omnisearch.py,sha256=7HVzz8LesS3Yo6exdlPf4WdPbi2DQekPR2CmkJcfBbI,10115
-linkarchivetools/utils/reflected.py,sha256=J1pYN8ekZC1pkhCRPzXy_ca2iVp2aED_5ZloVTnedzM,15878
-linkarchivetools-0.1.10.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-linkarchivetools-0.1.10.dist-info/METADATA,sha256=QKonY966i81YzUDw1PMS_AmHjpINrm1IL8dyr6FPORo,1203
-linkarchivetools-0.1.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-linkarchivetools-0.1.10.dist-info/RECORD,,
+linkarchivetools/utils/reflected.py,sha256=rpwCHX4rUvvyK_MqwmF8Z7eIqKqBK91xtcVbiqgVh1U,15445
+linkarchivetools-0.1.11.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+linkarchivetools-0.1.11.dist-info/METADATA,sha256=lewUvXEdmdQXDlad0t02cPy5PXv1oKItFKUUWNF7nhA,1254
+linkarchivetools-0.1.11.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+linkarchivetools-0.1.11.dist-info/RECORD,,

{linkarchivetools-0.1.10.dist-info → linkarchivetools-0.1.11.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 2.1.2
 Root-Is-Purelib: true
 Tag: py3-none-any