ingestify 0.6.2__py3-none-any.whl → 0.6.3__py3-none-any.whl

ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.6.2"
+__version__ = "0.6.3"

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -152,7 +152,13 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
     def dialect(self) -> Dialect:
         return self.session_provider.dialect
 
-    def _upsert(self, connection: Connection, table: Table, entities: list[dict]):
+    def _upsert(
+        self,
+        connection: Connection,
+        table: Table,
+        entities: list[dict],
+        immutable_rows: bool = False,
+    ):
         dialect = self.dialect.name
         if dialect == "mysql":
             from sqlalchemy.dialects.mysql import insert
@@ -167,13 +173,18 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         primary_key_columns = [column for column in table.columns if column.primary_key]
 
-        set_ = {
-            name: getattr(stmt.excluded, name)
-            for name, column in table.columns.items()
-            if column not in primary_key_columns
-        }
-
-        stmt = stmt.on_conflict_do_update(index_elements=primary_key_columns, set_=set_)
+        if immutable_rows:
+            stmt = stmt.on_conflict_do_nothing(index_elements=primary_key_columns)
+        else:
+            set_ = {
+                name: getattr(stmt.excluded, name)
+                for name, column in table.columns.items()
+                if column not in primary_key_columns
+            }
+
+            stmt = stmt.on_conflict_do_update(
+                index_elements=primary_key_columns, set_=set_
+            )
 
         connection.execute(stmt)
 
@@ -476,8 +487,10 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         with self.connect() as connection:
             try:
                 self._upsert(connection, dataset_table, datasets_entities)
-                self._upsert(connection, revision_table, revision_entities)
-                self._upsert(connection, file_table, file_entities)
+                self._upsert(
+                    connection, revision_table, revision_entities, immutable_rows=True
+                )
+                self._upsert(connection, file_table, file_entities, immutable_rows=True)
             except Exception:
                 connection.rollback()
                 raise

{ingestify-0.6.2.dist-info → ingestify-0.6.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.6.2
+Version: 0.6.3
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.6.2.dist-info → ingestify-0.6.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ingestify/__init__.py,sha256=2dkib_HJRMl-yqXrVrpQ9bLZmvrOFGDialtsAgYnAjI,301
+ingestify/__init__.py,sha256=CuCiGM6aFbNz1p2VC92CWR5tn9t7b_2nWoW7dlO4ADQ,301
 ingestify/cmdline.py,sha256=JcveX6e4i6mJtIllhTuruwbqxyoKIITIWE8kB6byvJU,7721
 ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
 ingestify/main.py,sha256=yYKA-4WAk04RdBCGmatsCKiPFQzpyufoG4VzHiWkVtU,8979
@@ -64,7 +64,7 @@ ingestify/infra/source/wyscout.py,sha256=DxCzdkzYpVRHTfV9GpF8pe3FzwIk-WHYUlea6nO
 ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
 ingestify/infra/store/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
-ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=aC7vzK8HKQLw5I2TNyqYQ6qYDpazn6OkukS73Jr15qI,19477
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=8zMPA4jeAPNLXZ2zzpG9SClto8ysqgA341JYsZPQNHg,19812
 ingestify/infra/store/dataset/sqlalchemy/tables.py,sha256=kALM32jbdeZ4Wn9gON-w2WSb5tH1lIWaBFgn5i29qTk,10635
 ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
 ingestify/infra/store/file/dummy_file_repository.py,sha256=azUq9c43Mz9-GWk9j0E97BaqyUKu-ZMrcuaIednLq5E,723
@@ -80,8 +80,8 @@ ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
 ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
 ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
-ingestify-0.6.2.dist-info/METADATA,sha256=4MoOdAoHVSAVDAZDj0ovLbbAAqJEScRJVpjrXrsvKjQ,18854
-ingestify-0.6.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-ingestify-0.6.2.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.6.2.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.6.2.dist-info/RECORD,,
+ingestify-0.6.3.dist-info/METADATA,sha256=-EpbTNkY4-qoUXCvfnM4Lpe_qdmNXGLm8SzL20kQ1AE,18854
+ingestify-0.6.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+ingestify-0.6.3.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.6.3.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.6.3.dist-info/RECORD,,