ingestify 0.6.2__tar.gz → 0.6.4__tar.gz

{ingestify-0.6.2 → ingestify-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.6.2
+Version: 0.6.4
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl

{ingestify-0.6.2 → ingestify-0.6.4}/ingestify/__init__.py

@@ -8,4 +8,4 @@ if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
 
-__version__ = "0.6.2"
+__version__ = "0.6.4"

{ingestify-0.6.2 → ingestify-0.6.4}/ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -152,7 +152,13 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
     def dialect(self) -> Dialect:
         return self.session_provider.dialect
 
-    def _upsert(self, connection: Connection, table: Table, entities: list[dict]):
+    def _upsert(
+        self,
+        connection: Connection,
+        table: Table,
+        entities: list[dict],
+        immutable_rows: bool = False,
+    ):
         dialect = self.dialect.name
         if dialect == "mysql":
             from sqlalchemy.dialects.mysql import insert
@@ -167,13 +173,18 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
 
         primary_key_columns = [column for column in table.columns if column.primary_key]
 
-        set_ = {
-            name: getattr(stmt.excluded, name)
-            for name, column in table.columns.items()
-            if column not in primary_key_columns
-        }
-
-        stmt = stmt.on_conflict_do_update(index_elements=primary_key_columns, set_=set_)
+        if immutable_rows:
+            stmt = stmt.on_conflict_do_nothing(index_elements=primary_key_columns)
+        else:
+            set_ = {
+                name: getattr(stmt.excluded, name)
+                for name, column in table.columns.items()
+                if column not in primary_key_columns
+            }
+
+            stmt = stmt.on_conflict_do_update(
+                index_elements=primary_key_columns, set_=set_
+            )
 
         connection.execute(stmt)
 
@@ -257,33 +268,35 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
             if not selectors:
                 raise ValueError("Selectors must contain at least one item")
 
-            attribute_cte = self._build_cte(
-                [selector.filtered_attributes for selector in selectors], "attributes"
-            )
-
-            keys = list(selectors[0].filtered_attributes.keys())
             first_selector = selectors[0].filtered_attributes
+            keys = list(first_selector.keys())
 
-            join_conditions = []
-            for k in keys:
-                if dialect == "postgresql":
-                    column = dataset_table.c.identifier[k]
+            if keys:
+                attribute_cte = self._build_cte(
+                    [selector.filtered_attributes for selector in selectors],
+                    "attributes",
+                )
 
-                    # Take the value from the first selector to determine the type.
-                    # TODO: check all selectors to determine the type
-                    v = first_selector[k]
-                    if isinstance(v, int):
-                        column = column.as_integer()
+                join_conditions = []
+                for k in keys:
+                    if dialect == "postgresql":
+                        column = dataset_table.c.identifier[k]
+
+                        # Take the value from the first selector to determine the type.
+                        # TODO: check all selectors to determine the type
+                        v = first_selector[k]
+                        if isinstance(v, int):
+                            column = column.as_integer()
+                        else:
+                            column = column.as_string()
                     else:
-                        column = column.as_string()
-                else:
-                    column = func.json_extract(dataset_table.c.identifier, f"$.{k}")
+                        column = func.json_extract(dataset_table.c.identifier, f"$.{k}")
 
-                join_conditions.append(attribute_cte.c[k] == column)
+                    join_conditions.append(attribute_cte.c[k] == column)
 
-            query = query.select_from(
-                dataset_table.join(attribute_cte, and_(*join_conditions))
-            )
+                query = query.select_from(
+                    dataset_table.join(attribute_cte, and_(*join_conditions))
+                )
 
         if where:
             query = query.filter(text(where))
@@ -476,8 +489,10 @@ class SqlAlchemyDatasetRepository(DatasetRepository):
         with self.connect() as connection:
             try:
                 self._upsert(connection, dataset_table, datasets_entities)
-                self._upsert(connection, revision_table, revision_entities)
-                self._upsert(connection, file_table, file_entities)
+                self._upsert(
+                    connection, revision_table, revision_entities, immutable_rows=True
+                )
+                self._upsert(connection, file_table, file_entities, immutable_rows=True)
             except Exception:
                 connection.rollback()
                 raise

{ingestify-0.6.2 → ingestify-0.6.4}/ingestify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.6.2
+Version: 0.6.4
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl