r2gg 3.1.4__py3-none-any.whl → 3.1.6__py3-none-any.whl

r2gg/__about__.py

@@ -34,7 +34,7 @@ __uri_repository__ = "https://github.com/IGNF/route-graph-generator/"
 __uri_tracker__ = f"{__uri_repository__}issues/"
 __uri__ = __uri_repository__
 
-__version__ = "3.1.4"
+__version__ = "3.1.6"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

r2gg/_database.py

@@ -23,7 +23,7 @@ def database_retry_decorator(func):
             except (OperationalError, DatabaseError, InterfaceError) as e:
                 if attempt >= RETRY:
                     self.logger.error(f"Query failed after {RETRY} attempts: {str(e).rstrip()}")
-                    return
+                    raise e
 
                 self.logger.error(
                     f"Attempt {attempt}/{RETRY} failed ({str(e).rstrip()}), retrying in {DELAY} seconds"
@@ -108,49 +108,60 @@ class DatabaseManager:
             time.sleep(DELAY)
             self._connection = self.connect_working_db()
 
-    def execute_select_query(self, cursor, query, show_duration):
-        if TIMEOUT:
-            cursor.execute("SET statement_timeout = %s", (1000 * TIMEOUT,))  # timeout in milliseconds
-
-        if show_duration:
-            self.logger.info("SQL: {}".format(query))
-            st_execute = time.time()
-            cursor.execute(query)
-            et_execute = time.time()
-            self.logger.info("Execution ended. Elapsed time : %s seconds." % (et_execute - st_execute))
-        else:
-            cursor.execute(query)
-
-    @database_retry_decorator
+    # IMPORTANT:
+    # Streaming SELECTs must NOT use retry logic.
+    # If the connection drops, the cursor state is unrecoverable.
     def execute_select_fetch_multiple(self, query, batchsize=1, show_duration=False):
-        with self._connection.cursor(cursor_factory=DictCursor) as cursor:
-            self.execute_select_query(cursor, query, show_duration)
-            rows = cursor.fetchmany(batchsize)
+        """
+        Streaming SELECT using a named server-side cursor.
+        No retry. No reconnect. No commit.
+        Fail fast if the connection drops (old behavior).
+        """
+        self.ensure_connection()
+        cursor_name = f"cursor_{int(time.time() * 1000)}"
+        with self._connection.cursor(cursor_factory=DictCursor, name=cursor_name) as cursor:
+            if TIMEOUT:
+                cursor.execute("SET statement_timeout = %s", (1000 * TIMEOUT,))
+            if show_duration:
+                self.logger.info(f"SQL: {query}")
+                st = time.time()
+                cursor.execute(query)
+                self.logger.info(
+                    "Execution ended. Elapsed time : %s seconds.",
+                    time.time() - st
+                )
+            else:
+                cursor.execute(query)
+
             count = cursor.rowcount
-            while rows:
+
+            while True:
+                rows = cursor.fetchmany(batchsize)
+                if not rows:
+                    break
                 if batchsize == 1:
                     rows = rows.pop()
                 yield rows, count
-                rows = cursor.fetchmany(batchsize)
-            self._connection.commit()
-            return
 
     # the method below should be used as a generator function otherwise use execute_update
     @database_retry_decorator
     def execute_update_query(self, query, params=None, isolation_level=None, show_duration=False):
+        self.ensure_connection()
         if show_duration :
             self.logger.info("SQL: {}".format(query))
         st_execute = time.time()
         with self._connection.cursor(cursor_factory=DictCursor) as cursor:
             old_isolation_level = self._connection.isolation_level
-            if isolation_level is not None:
-                self._connection.set_isolation_level(isolation_level)
-            cursor.execute(query, params)
-            self._connection.commit()
+            try:
+                if isolation_level is not None:
+                    self._connection.set_isolation_level(isolation_level)
+                cursor.execute(query, params)
+                self._connection.commit()
+            finally:
+                self._connection.set_isolation_level(old_isolation_level)
             if show_duration:
                 et_execute = time.time()
                 self.logger.info("Execution ended. Elapsed time : %s seconds." % (et_execute - st_execute))
-            self._connection.set_isolation_level(old_isolation_level)
         yield  # the decorator database_retry_decorator only supports generators
         return
 
@@ -158,6 +169,9 @@ class DatabaseManager:
         next(self.execute_update_query(query, params=params, isolation_level=isolation_level), None)
 
     def execute_select_fetch_one(self, query, show_duration=False):
-        gen = self.execute_select_fetch_multiple(query, 1, show_duration)
-        row, count = next(gen, (None, None))
-        return row, count
+        try:
+            gen = self.execute_select_fetch_multiple(query, 1, show_duration)
+            row, count = next(gen, (None, None))
+            return row, count
+        finally:
+            gen.close()  # Ensure the generator is closed to free resources

r2gg/_pivot_to_osm.py

@@ -81,10 +81,13 @@ def pivot_to_osm(config, source, db_configs, database: DatabaseManager, logger,
                     sql_query_nodes += " LIMIT {} OFFSET {}".format(batchsize, offset)
                     offset += batchsize
                     logger.info("Writing nodes")
-                    for row, count in database.execute_select_fetch_multiple(sql_query_nodes, show_duration=True):
-                        nodeEl = writeNode(row, extraction_date)
-                        xf.write(nodeEl, pretty_print=True)
-
+                    gen = database.execute_select_fetch_multiple(sql_query_nodes, show_duration=True)
+                    try:
+                        for row, count in gen:
+                            nodeEl = writeNode(row, extraction_date)
+                            xf.write(nodeEl, pretty_print=True)
+                    finally:
+                        gen.close()
                     logger.info("%s / %s nodes ajoutés" % (offset, nodesize))
                 et_nodes = time.time()
                 logger.info("Writing nodes ended. Elapsed time : %s seconds." % (et_nodes - st_nodes))
@@ -124,16 +127,19 @@ def pivot_to_osm(config, source, db_configs, database: DatabaseManager, logger,
                 logger.info("Writing restrictions")
                 st_execute = time.time()
                 i = 1
-                for row, count in database.execute_select_fetch_multiple(sql_query_non_comm, show_duration=True):
-                    if row['common_vertex_id'] == -1:
+                gen = database.execute_select_fetch_multiple(sql_query_non_comm, show_duration=True)
+                try:
+                    for row, count in gen:
+                        if row['common_vertex_id'] == -1:
+                            i += 1
+                            continue
+                        ResEl = writeRes(row, i, extraction_date)
+                        xf.write(ResEl, pretty_print=True)
+                        if count > 0 and (i % ceil(count / 10) == 0):
+                            logger.info("%s / %s restrictions ajoutés" % (i, count))
                         i += 1
-                        continue
-                    ResEl = writeRes(row, i, extraction_date)
-                    xf.write(ResEl, pretty_print=True)
-                    if (i % ceil(count / 10) == 0):
-                        logger.info("%s / %s restrictions ajoutés" % (i, count))
-                    i += 1
-
+                finally:
+                    gen.close()
                 et_execute = time.time()
                 logger.info("Writing restrictions ended. Elapsed time : %s seconds." % (et_execute - st_execute))
 

r2gg/_pivot_to_pgr.py

@@ -115,38 +115,41 @@ def pivot_to_pgr(source, cost_calculation_file_path, database_work: DatabaseMana
     index = 0
     batchsize = 10000
     generator = database_work.execute_select_fetch_multiple(tr_query, show_duration=True, batchsize=batchsize)
-    rows, count = next(generator,(None, None))
-    # Insertion petit à petit -> plus performant
-
-    logger.info("SQL: Inserting or updating {} values in out db".format(count))
-
-    st_execute = time.time()
-
-    while rows:
-        values_str = ""
-        # Tuple des valuers à insérer
-        values_tuple = ()
-        for row in rows:
-            values_str += "(%s, %s, %s),"
-            values_tuple += (index, row['id_from'], row['id_to'])
-            index += 1
-        values_str = values_str[:-1]
-
-        set_on_conflict = (
-            "id_from = excluded.id_from,id_to = excluded.id_to"
-        )
+    try:
+        rows, count = next(generator,(None, None))
+        # Insertion petit à petit -> plus performant
+
+        logger.info("SQL: Inserting or updating {} values in out db".format(count))
+
+        st_execute = time.time()
+
+        while rows:
+            values_str = ""
+            # Tuple des valuers à insérer
+            values_tuple = ()
+            for row in rows:
+                values_str += "(%s, %s, %s),"
+                values_tuple += (index, row['id_from'], row['id_to'])
+                index += 1
+            values_str = values_str[:-1]
+
+            set_on_conflict = (
+                "id_from = excluded.id_from,id_to = excluded.id_to"
+            )
 
-        sql_insert = """
-            INSERT INTO {}.turn_restrictions (id, id_from, id_to)
-            VALUES {}
-            ON CONFLICT (id) DO UPDATE
-              SET {};
-        """.format(schema, values_str, set_on_conflict)
-        database_out.execute_update(sql_insert, values_tuple)
+            sql_insert = """
+                INSERT INTO {}.turn_restrictions (id, id_from, id_to)
+                VALUES {}
+                ON CONFLICT (id) DO UPDATE
+                SET {};
+            """.format(schema, values_str, set_on_conflict)
+            database_out.execute_update(sql_insert, values_tuple)
 
-        rows, _ = next(generator,(None, None))
+            rows, _ = next(generator,(None, None))
 
-    et_execute = time.time()
+        et_execute = time.time()
+    finally:
+        generator.close()  # Ensure the generator is closed to free resources
     logger.info("Writing turn restrinctions Done. Elapsed time : %s seconds." %(et_execute - st_execute))
 
     # Noeuds ---------------------------------------------------------------------------------------
@@ -171,30 +174,32 @@ def pivot_to_pgr(source, cost_calculation_file_path, database_work: DatabaseMana
     index = 0
     batchsize = 10000
     generator = database_work.execute_select_fetch_multiple(nd_query, show_duration=True, batchsize=batchsize)
-    rows, count = next(generator, (None, None))
-    while rows:
-        values_str = ""
-        # Tuple des valeurs à insérer
-        values_tuple = ()
-        for row in rows:
-            values_str += "(%s, %s),"
-            values_tuple += (row['id'], row['geom'])
-            index += 1
-        values_str = values_str[:-1]
-
-        set_on_conflict = (
-            "the_geom = excluded.the_geom"
-        )
-
-        sql_insert = """
-            INSERT INTO {}_vertices_pgr (id, the_geom)
-            VALUES {}
-            ON CONFLICT (id) DO UPDATE
-              SET {};
-        """.format(ways_table_name, values_str, set_on_conflict)
-        database_out.execute_update(sql_insert, values_tuple)
-        rows, _ = next(generator,(None, None))
+    try:
+        rows, count = next(generator, (None, None))
+        while rows:
+            values_str = ""
+            # Tuple des valeurs à insérer
+            values_tuple = ()
+            for row in rows:
+                values_str += "(%s, %s),"
+                values_tuple += (row['id'], row['geom'])
+                index += 1
+            values_str = values_str[:-1]
+
+            set_on_conflict = (
+                "the_geom = excluded.the_geom"
+            )
 
+            sql_insert = """
+                INSERT INTO {}_vertices_pgr (id, the_geom)
+                VALUES {}
+                ON CONFLICT (id) DO UPDATE
+                SET {};
+            """.format(ways_table_name, values_str, set_on_conflict)
+            database_out.execute_update(sql_insert, values_tuple)
+            rows, _ = next(generator,(None, None))
+    finally:
+        generator.close()  # Ensure the generator is closed to free resources
 
     et_execute = time.time()
     logger.info("Writing vertices Done. Elapsed time : %s seconds." %(et_execute - st_execute))
@@ -266,45 +271,48 @@ def pivot_to_pgr(source, cost_calculation_file_path, database_work: DatabaseMana
     # logger.info("SQL: Inserting or updating {} values in out db".format(cursor_in.rowcount))
     st_execute = time.time()
     percent = 0
-    rows, count = next(generator, (None, None))
-    while rows:
-        percent += 1000000 / count
-        # Chaîne permettant l'insertion de valeurs via psycopg
-        values_str = ""
-        # Tuple des valuers à insérer
-        values_tuple = ()
-        for row in rows:
-            values_str += "(" + single_value_str + "),"
-            output_costs = output_costs_from_costs_config(costs, row)
-            values_tuple += tuple(
-                row[ output_columns_name ] for output_columns_name in output_columns_names
-            ) + output_costs
-        values_str = values_str[:-1]
-
-        output_columns = "("
-        for output_columns_name in output_columns_names:
-            output_columns += output_columns_name + ','
-        output_columns = output_columns[:-1]
-
-        set_on_conflict = ''
-        for output_columns_name in output_columns_names:
-            set_on_conflict += "{0} = excluded.{0},".format(output_columns_name)
-        set_on_conflict = set_on_conflict[:-1]
-
-        for output in costs["outputs"]:
-            output_columns += "," + output["name"] + ",reverse_" + output["name"]
-            set_on_conflict += ",{0} = excluded.{0}".format(output["name"])
-            set_on_conflict += ",{0} = excluded.{0}".format("reverse_" + output["name"])
-
-        output_columns += ")"
-        sql_insert = """
-            INSERT INTO {} {}
-            VALUES {}
-            ON CONFLICT (id) DO UPDATE
-              SET {};
-            """.format(ways_table_name, output_columns, values_str, set_on_conflict)
-        database_out.execute_update(sql_insert, values_tuple)
-        rows, _ = next(generator,(None, None))
+    try:
+        rows, count = next(generator, (None, None))
+        while rows:
+            percent += 1000000 / count
+            # Chaîne permettant l'insertion de valeurs via psycopg
+            values_str = ""
+            # Tuple des valuers à insérer
+            values_tuple = ()
+            for row in rows:
+                values_str += "(" + single_value_str + "),"
+                output_costs = output_costs_from_costs_config(costs, row)
+                values_tuple += tuple(
+                    row[ output_columns_name ] for output_columns_name in output_columns_names
+                ) + output_costs
+            values_str = values_str[:-1]
+
+            output_columns = "("
+            for output_columns_name in output_columns_names:
+                output_columns += output_columns_name + ','
+            output_columns = output_columns[:-1]
+
+            set_on_conflict = ''
+            for output_columns_name in output_columns_names:
+                set_on_conflict += "{0} = excluded.{0},".format(output_columns_name)
+            set_on_conflict = set_on_conflict[:-1]
+
+            for output in costs["outputs"]:
+                output_columns += "," + output["name"] + ",reverse_" + output["name"]
+                set_on_conflict += ",{0} = excluded.{0}".format(output["name"])
+                set_on_conflict += ",{0} = excluded.{0}".format("reverse_" + output["name"])
+
+            output_columns += ")"
+            sql_insert = """
+                INSERT INTO {} {}
+                VALUES {}
+                ON CONFLICT (id) DO UPDATE
+                SET {};
+                """.format(ways_table_name, output_columns, values_str, set_on_conflict)
+            database_out.execute_update(sql_insert, values_tuple)
+            rows, _ = next(generator,(None, None))
+    finally:
+        generator.close()  # Ensure the generator is closed to free resources
 
     et_execute = time.time()
     logger.info("Writing ways ended. Elapsed time : %s seconds." %(et_execute - st_execute))

r2gg/cli.py

@@ -13,20 +13,26 @@ from r2gg._database import DatabaseManager
 def sql2pivot():
     config, resource, db_configs, logger = configure()
     database = DatabaseManager(db_configs[config["workingSpace"]["baseId"]], logger)
-    sql_convert(config, resource, db_configs, database, logger)
-    database.disconnect_working_db()
+    try:
+        sql_convert(config, resource, db_configs, database, logger)
+    finally:
+        database.disconnect_working_db()
 
 def pivot2pgrouting():
     config, resource, db_configs, logger = configure()
     database = DatabaseManager(db_configs[config["workingSpace"]["baseId"]], logger)
-    pgr_convert(resource, db_configs, database, logger)
-    database.disconnect_working_db()
+    try:
+        pgr_convert(resource, db_configs, database, logger)
+    finally:
+        database.disconnect_working_db()
 
 def pivot2osm():
     config, resource, db_configs, logger = configure()
     database = DatabaseManager(db_configs[config["workingSpace"]["baseId"]], logger)
-    osm_convert(config, resource, db_configs, database, logger)
-    database.disconnect_working_db()
+    try:
+        osm_convert(config, resource, db_configs, database, logger)
+    finally:
+        database.disconnect_working_db()
 
 def osm2osrm():
     config, resource, _, logger = configure()

{r2gg-3.1.4.dist-info → r2gg-3.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: r2gg
-Version: 3.1.4
+Version: 3.1.6
 Summary: Route Graph Generator (r2gg) est un script Python qui permet la génération de graphes pour des moteurs de calcul d'itinéraire
 Home-page: https://github.com/IGNF/route-graph-generator/
 Author: IGNF

{r2gg-3.1.4.dist-info → r2gg-3.1.6.dist-info}/RECORD

@@ -1,7 +1,7 @@
-r2gg/__about__.py,sha256=VjF-b03qrVBDGQ-KCnD7lEiODARNWwvwp8KVejqA1Cc,1343
+r2gg/__about__.py,sha256=UNywC946fLmBM6E4ZaK0EwaEVTuS0iZofMNu7wYKC6I,1343
 r2gg/__init__.py,sha256=zERkbHrNMbNZleuOxsfA7afQExXxpP7-IWVP1IFzNNs,88
 r2gg/_configure.py,sha256=HGLvnHa_XUV_BGBL_Hn-llaRDH6JK2svdXK6vo61_GU,2405
-r2gg/_database.py,sha256=xXBkxGyjNRRJzIGUKDhB5OWq0G_FV6KlB5uHJnOXb_c,6306
+r2gg/_database.py,sha256=Mz8zx2lVkChNoi4Nxxz7sQ59TiYzjKeSLhAGYiFMKDw,6733
 r2gg/_file_copier.py,sha256=NzDd6ZqxQsQ_EulXg6jZxECUMjlJ0VBtr40-c1Rde-w,578
 r2gg/_lua_builder.py,sha256=W00BWFkRoKSyBgcWCl00hxBNqvE3Yf7Aw6DeLLwfO5M,10420
 r2gg/_main.py,sha256=vxXsSaocfeQfMDvamiRCEJPESwygMs47LQU25gHpHSY,22368
@@ -9,16 +9,16 @@ r2gg/_osm_building.py,sha256=MFFY3EYOVO4rPsYk90Oda70KH1DKW2rlRj8XT4Vx_4U,4227
 r2gg/_osm_to_pbf.py,sha256=ZCGaqao4r2xkwIvKjQSOBd7TVXzO_KDAT2PptnCuAoY,1088
 r2gg/_output_costs_from_costs_config.py,sha256=g7Qy4tlUIOA7DJcwXW9ZxhcOJ5Srq1c8hufylI2e0m8,6208
 r2gg/_path_converter.py,sha256=CTtFHucxTSDB89TMbTO7Q0VqRCs2295GTA4_3cftUVc,340
-r2gg/_pivot_to_osm.py,sha256=hucBK63QJSkJxNv9bfcNPRs0Ky0vIolccELrGPq3a2s,6933
-r2gg/_pivot_to_pgr.py,sha256=39dLpl0_HG-0RLp4vlRqwhXBOzdkxaYhp3SdXxS869k,15469
+r2gg/_pivot_to_osm.py,sha256=fCpimwu7c76IQKYWL9IdTmUv5stBtBBCsAWQTn4T8S0,7217
+r2gg/_pivot_to_pgr.py,sha256=YhHJbqJOHMKeDjPKT0i6cgHSuMbLF_yAw6A1ZLHvIbE,16086
 r2gg/_read_config.py,sha256=VQ6d6Oi1aKHwwURA3KrbpWHK0zP18aJ98XQJiHZb8oI,721
 r2gg/_sql_building.py,sha256=DefYIerZ0k_yltgJFaSzTXAZPSSUH7WAxN3d5FceoWw,840
 r2gg/_subprocess_execution.py,sha256=ipsynsXohonN9YfmxX7DFTFCvJWsvrLDuA5Y-0QDUDM,1603
 r2gg/_valhalla_lua_builder.py,sha256=vNUnsu4yYQflnNv49-ElxrfIqa4b30vJP740ab7M7rY,13816
-r2gg/cli.py,sha256=jeMiLc1UVAhNFNe_p8nRnyF8V_7kliqIzJsLG1MKpIo,2174
-r2gg-3.1.4.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-r2gg-3.1.4.dist-info/METADATA,sha256=3nyIXee9UgSuU5CvmAisLZekI0jBS9B0jVbjFRTUivU,5199
-r2gg-3.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-r2gg-3.1.4.dist-info/entry_points.txt,sha256=Km7XbbcVI9jy1TQI0raQfRyKdNGAUIwVkkBG5YeQP4k,275
-r2gg-3.1.4.dist-info/top_level.txt,sha256=fj9IaWXORCdMRcnWWZ7LmvOG4W_sVH8J4BUWf1pM37c,5
-r2gg-3.1.4.dist-info/RECORD,,
+r2gg/cli.py,sha256=UKzYyOeUNZOeP-mjMOZaNviBkOA4fCqhB1xYMKXCWrk,2264
+r2gg-3.1.6.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+r2gg-3.1.6.dist-info/METADATA,sha256=jeSbamu53xn4vTMEhvJDGy7yviGYIzm7Nz3O6R50QYE,5199
+r2gg-3.1.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+r2gg-3.1.6.dist-info/entry_points.txt,sha256=Km7XbbcVI9jy1TQI0raQfRyKdNGAUIwVkkBG5YeQP4k,275
+r2gg-3.1.6.dist-info/top_level.txt,sha256=fj9IaWXORCdMRcnWWZ7LmvOG4W_sVH8J4BUWf1pM37c,5
+r2gg-3.1.6.dist-info/RECORD,,

{r2gg-3.1.4.dist-info → r2gg-3.1.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any