r2gg 2.2.6__py3-none-any.whl → 3.1.4__py3-none-any.whl

r2gg/_pivot_to_osm.py

@@ -1,17 +1,17 @@
-from datetime import date
-from math import ceil
 import os
 import time
+from datetime import date
+from math import ceil
 
 from lxml import etree
-from psycopg2.extras import DictCursor
 
 from r2gg._osm_building import writeNode, writeWay, writeWayNds, writeRes, writeWayTags
-from r2gg._sql_building import getQueryByTableAndBoundingBox
 from r2gg._osm_to_pbf import osm_to_pbf
+from r2gg._sql_building import getQueryByTableAndBoundingBox
+from r2gg._database import DatabaseManager
 
 
-def pivot_to_osm(config, source, db_configs, connection, logger, output_is_pbf = False):
+def pivot_to_osm(config, source, db_configs, database: DatabaseManager, logger, output_is_pbf=False):
     """
     Fonction de conversion depuis la bdd pivot vers le fichier osm puis pbf le cas échéant
 
@@ -22,11 +22,10 @@ def pivot_to_osm(config, source, db_configs, connection, logger, output_is_pbf =
     source: dict
     db_configs: dict
         dictionnaire correspondant aux configurations des bdd
-    connection: psycopg2.connection
-        connection à la bdd de travail
+    database: r2gg.DatabaseManager
+        gestionnaire de connexion et d'exécution de la base de la bdd
     logger: logging.Logger
     """
-
     logger.info("Convert pivot to OSM format for a source")
 
     # Récupération de la date d'extraction
@@ -44,16 +43,14 @@ def pivot_to_osm(config, source, db_configs, connection, logger, output_is_pbf =
     source_db_config = db_configs[source['mapping']['source']['baseId']]
     input_schema = source_db_config.get('schema')
 
-    cursor = connection.cursor(cursor_factory=DictCursor)
-
-    logger.info(f"SQL: select last_value from {input_schema}.nodes_id_seq")
-    cursor.execute(f"select last_value from {input_schema}.nodes_id_seq")
-    vertexSequence = cursor.fetchone()[0]
+    last_value_nodes_query = f"select last_value from {input_schema}.nodes_id_seq"
+    vertexSequence, _ = database.execute_select_fetch_one(last_value_nodes_query, show_duration=True)
+    vertexSequence = vertexSequence[0]
     logger.info(vertexSequence)
 
-    logger.info(f"SQL: select last_value from {input_schema}.edges_id_seq")
-    cursor.execute(f"select last_value from {input_schema}.edges_id_seq")
-    edgeSequence = cursor.fetchone()[0]
+    last_value_edges_query = f"select last_value from {input_schema}.edges_id_seq"
+    edgeSequence, _ = database.execute_select_fetch_one(last_value_edges_query, show_duration=True)
+    edgeSequence = edgeSequence[0]
     logger.info(edgeSequence)
 
     logger.info("Starting conversion from pivot to OSM")
@@ -70,13 +67,8 @@ def pivot_to_osm(config, source, db_configs, connection, logger, output_is_pbf =
             with xf.element("osm", attribs):
 
                 # Récupération du nombre de nodes
-                sql_query = f"SELECT COUNT(*) as cnt FROM {input_schema}.nodes"
-                logger.info("SQL: {}".format(sql_query))
-                st_execute = time.time()
-                cursor.execute(sql_query)
-                et_execute = time.time()
-                logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-                row = cursor.fetchone()
+                number_of_nodes_query = f"SELECT COUNT(*) as cnt FROM {input_schema}.nodes"
+                row, _ = database.execute_select_fetch_one(number_of_nodes_query, show_duration=True)
                 nodesize = row["cnt"]
 
                 # Ecriture des nodes
@@ -85,34 +77,21 @@ def pivot_to_osm(config, source, db_configs, connection, logger, output_is_pbf =
                 logger.info(f"Writing nodes: {nodesize} ways to write")
                 st_nodes = time.time()
                 while offset < nodesize:
-                  sql_query = getQueryByTableAndBoundingBox(f'{input_schema}.nodes', source['bbox'])
-                  sql_query += " LIMIT {} OFFSET {}".format(batchsize, offset)
-                  logger.info("SQL: {}".format(sql_query))
-                  st_execute = time.time()
-                  cursor.execute(sql_query)
-                  et_execute = time.time()
-                  offset += batchsize
-                  logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-                  row = cursor.fetchone()
-                  logger.info("Writing nodes")
-                  st_execute = time.time()
-                  i = 1
-                  while row:
-                      nodeEl = writeNode(row, extraction_date)
-                      xf.write(nodeEl, pretty_print=True)
-                      row = cursor.fetchone()
-                  logger.info("%s / %s nodes ajoutés" %(offset, nodesize))
+                    sql_query_nodes = getQueryByTableAndBoundingBox(f'{input_schema}.nodes', source['bbox'])
+                    sql_query_nodes += " LIMIT {} OFFSET {}".format(batchsize, offset)
+                    offset += batchsize
+                    logger.info("Writing nodes")
+                    for row, count in database.execute_select_fetch_multiple(sql_query_nodes, show_duration=True):
+                        nodeEl = writeNode(row, extraction_date)
+                        xf.write(nodeEl, pretty_print=True)
+
+                    logger.info("%s / %s nodes ajoutés" % (offset, nodesize))
                 et_nodes = time.time()
-                logger.info("Writing nodes ended. Elapsed time : %s seconds." %(et_nodes - st_nodes))
+                logger.info("Writing nodes ended. Elapsed time : %s seconds." % (et_nodes - st_nodes))
 
                 # Récupération du nombre de ways
-                sql_query = f"SELECT COUNT(*) as cnt FROM {input_schema}.edges"
-                logger.info("SQL: {}".format(sql_query))
-                st_execute = time.time()
-                cursor.execute(sql_query)
-                et_execute = time.time()
-                logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-                row = cursor.fetchone()
+                sql_query_edges_count = f"SELECT COUNT(*) as cnt FROM {input_schema}.edges"
+                row, _ = database.execute_select_fetch_one(sql_query_edges_count, show_duration=True)
                 edgesize = row["cnt"]
 
                 # Ecriture des ways
@@ -121,64 +100,49 @@ def pivot_to_osm(config, source, db_configs, connection, logger, output_is_pbf =
                 logger.info(f"Writing ways: {edgesize} ways to write")
                 st_edges = time.time()
                 while offset < edgesize:
-                  sql_query2 = getQueryByTableAndBoundingBox(f'{input_schema}.edges', source['bbox'], ['*', f'{input_schema}.inter_nodes(geom) as internodes'])
-                  sql_query2 += " LIMIT {} OFFSET {}".format(batchsize, offset)
-                  logger.info("SQL: {}".format(sql_query2))
-                  st_execute = time.time()
-                  cursor.execute(sql_query2)
-                  et_execute = time.time()
-                  offset += batchsize
-                  logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-                  row = cursor.fetchone()
-                  st_execute = time.time()
-                  i = 1
-                  while row:
-                      wayEl = writeWay(row, extraction_date)
-                      for node in row['internodes']:
-                          vertexSequence = vertexSequence + 1
-                          node['id'] = vertexSequence
-                          nodeEl = writeNode(node, extraction_date)
-                          xf.write(nodeEl, pretty_print=True)
-                      wayEl = writeWayNds(wayEl, row, row['internodes'])
-                      wayEl = writeWayTags(wayEl, row)
-                      xf.write(wayEl, pretty_print=True)
-                      row = cursor.fetchone()
-                  logger.info("%s / %s ways ajoutés" %(offset, edgesize))
+                    sql_query_edges = getQueryByTableAndBoundingBox(f'{input_schema}.edges', source['bbox'], ['*',
+                                                                                                              f'{input_schema}.inter_nodes(geom) as internodes'])
+                    sql_query_edges += " LIMIT {} OFFSET {}".format(batchsize, offset)
+                    offset += batchsize
+                    for row, count in database.execute_select_fetch_multiple(sql_query_edges, show_duration=True):
+                        wayEl = writeWay(row, extraction_date)
+                        for node in row['internodes']:
+                            vertexSequence = vertexSequence + 1
+                            node['id'] = vertexSequence
+                            nodeEl = writeNode(node, extraction_date)
+                            xf.write(nodeEl, pretty_print=True)
+                        wayEl = writeWayNds(wayEl, row, row['internodes'])
+                        wayEl = writeWayTags(wayEl, row)
+                        xf.write(wayEl, pretty_print=True)
+
+                    logger.info("%s / %s ways ajoutés" % (offset, edgesize))
                 et_edges = time.time()
-                logger.info("Writing ways ended. Elapsed time : %s seconds." %(et_edges - st_edges))
+                logger.info("Writing ways ended. Elapsed time : %s seconds." % (et_edges - st_edges))
 
                 # Ecriture des restrictions
-                sql_query3 = f"select * from {input_schema}.non_comm"
-                logger.info("SQL: {}".format(sql_query3))
-                st_execute = time.time()
-                cursor.execute(sql_query3)
-                et_execute = time.time()
-                logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-                row = cursor.fetchone()
+                sql_query_non_comm = f"select * from {input_schema}.non_comm"
                 logger.info("Writing restrictions")
                 st_execute = time.time()
                 i = 1
-                while row:
+                for row, count in database.execute_select_fetch_multiple(sql_query_non_comm, show_duration=True):
                     if row['common_vertex_id'] == -1:
-                        row = cursor.fetchone()
                         i += 1
                         continue
                     ResEl = writeRes(row, i, extraction_date)
                     xf.write(ResEl, pretty_print=True)
-                    row = cursor.fetchone()
-                    if (i % ceil(cursor.rowcount/10) == 0):
-                        logger.info("%s / %s restrictions ajoutés" %(i, cursor.rowcount))
+                    if (i % ceil(count / 10) == 0):
+                        logger.info("%s / %s restrictions ajoutés" % (i, count))
                     i += 1
+
                 et_execute = time.time()
-                logger.info("Writing restrictions ended. Elapsed time : %s seconds." %(et_execute - st_execute))
+                logger.info("Writing restrictions ended. Elapsed time : %s seconds." % (et_execute - st_execute))
 
     except etree.SerialisationError:
-        logger.warn("WARNING: XML file not closed properly (lxml.etree.SerialisationError)")
+        logger.warning("WARNING: XML file not closed properly (lxml.etree.SerialisationError)")
 
-    cursor.close()
     end_time = time.time()
-    logger.info("Conversion from pivot to OSM ended. Elapsed time : %s seconds." %(end_time - start_time))
+    logger.info("Conversion from pivot to OSM ended. Elapsed time : %s seconds." % (end_time - start_time))
 
     # osm2pbf : Gestion du format osm.pbf
     if output_is_pbf:
-        osm_to_pbf(filename, filename+'.pbf', logger)
+        osm_to_pbf(filename, filename + '.pbf', logger)

r2gg/_pivot_to_pgr.py

@@ -7,8 +7,9 @@ from psycopg2.extras import DictCursor
 from r2gg._output_costs_from_costs_config import output_costs_from_costs_config
 from r2gg._read_config import config_from_path
 from r2gg._sql_building import getQueryByTableAndBoundingBox
+from r2gg._database import DatabaseManager
 
-def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection_out, schema, input_schema, logger):
+def pivot_to_pgr(source, cost_calculation_file_path, database_work: DatabaseManager, database_out: DatabaseManager, schema, input_schema, logger):
     """
     Fonction de conversion depuis la bdd pivot vers la base pgr
 
@@ -17,9 +18,9 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
     source: dict
     cost_calculation_file_path: str
         chemin vers le fichier json de configuration des coûts
-    connection_work: psycopg2.connection
+    database_work: DatabaseManager
         connection à la bdd de travail
-    connection_out: psycopg2.connection
+    database_out: DatabaseManager
         connection à la bdd pgrouting de sortie
     schema: str
         nom du schéma dans la base de sortie
@@ -28,12 +29,10 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
     logger: logging.Logger
     """
 
-    cursor_in = connection_work.cursor(cursor_factory=DictCursor, name="cursor_in")
     ways_table_name = schema + '.ways'
     # Récupération des coûts à calculer
     costs = config_from_path(cost_calculation_file_path)
 
-    cursor_out = connection_out.cursor()
     # Création de la edge_table pgrouting
     create_table = """
         DROP TABLE IF EXISTS {0};
@@ -64,7 +63,6 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             nombre_de_voies text,
             insee_commune_gauche text,
             insee_commune_droite text,
-            bande_cyclable text,
             itineraire_vert boolean,
             sens_de_circulation text,
             reserve_aux_bus text,
@@ -79,10 +77,14 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             matieres_dangereuses_interdites boolean,
             cpx_gestionnaire text,
             cpx_numero_route_europeenne text,
-            cpx_classement_administratif text
+            cpx_classement_administratif text,
+            transport_exceptionnel boolean,
+            vla_par_defaut integer,
+            cout_penalites numeric,
+            vehicule_leger_interdit boolean,
+            cout_vehicule_prioritaire numeric
         );""".format(ways_table_name)
-    logger.debug("SQL: {}".format(create_table))
-    cursor_out.execute(create_table)
+    database_out.execute_update(create_table)
 
     # Ajout des colonnes de coûts
     add_columns = "ALTER TABLE {} ".format(ways_table_name)
@@ -91,7 +93,7 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
         add_columns += "ADD COLUMN IF NOT EXISTS {} double precision,".format("reverse_" + output["name"])
     add_columns = add_columns[:-1]
     logger.debug("SQL: adding costs columns \n {}".format(add_columns))
-    cursor_out.execute(add_columns)
+    database_out.execute_update(add_columns)
 
     logger.info("Starting conversion")
     start_time = time.time()
@@ -105,34 +107,30 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             id_from bigint,
             id_to bigint
     );""".format(schema)
-    logger.debug("SQL: {}".format(create_non_comm))
-    cursor_out.execute(create_non_comm)
+    database_out.execute_update(create_non_comm)
 
     logger.info("Populating turn restrictions")
     tr_query = f"SELECT id_from, id_to FROM {input_schema}.non_comm;"
 
-    logger.debug("SQL: {}".format(tr_query))
-    st_execute = time.time()
-    cursor_in.execute(tr_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-    # Insertion petit à petit -> plus performant
-    logger.info("SQL: Inserting or updating {} values in out db".format(cursor_in.rowcount))
-    st_execute = time.time()
     index = 0
     batchsize = 10000
-    rows = cursor_in.fetchmany(batchsize)
+    generator = database_work.execute_select_fetch_multiple(tr_query, show_duration=True, batchsize=batchsize)
+    rows, count = next(generator,(None, None))
+    # Insertion petit à petit -> plus performant
+
+    logger.info("SQL: Inserting or updating {} values in out db".format(count))
+
+    st_execute = time.time()
+
     while rows:
         values_str = ""
-        for row in rows:
-            values_str += "(%s, %s, %s),"
-        values_str = values_str[:-1]
-
         # Tuple des valuers à insérer
         values_tuple = ()
         for row in rows:
+            values_str += "(%s, %s, %s),"
             values_tuple += (index, row['id_from'], row['id_to'])
             index += 1
+        values_str = values_str[:-1]
 
         set_on_conflict = (
             "id_from = excluded.id_from,id_to = excluded.id_to"
@@ -144,17 +142,15 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             ON CONFLICT (id) DO UPDATE
               SET {};
         """.format(schema, values_str, set_on_conflict)
-        cursor_out.execute(sql_insert, values_tuple)
-        connection_out.commit()
-        rows = cursor_in.fetchmany(batchsize)
+        database_out.execute_update(sql_insert, values_tuple)
+
+        rows, _ = next(generator,(None, None))
 
     et_execute = time.time()
-    cursor_in.close()
     logger.info("Writing turn restrinctions Done. Elapsed time : %s seconds." %(et_execute - st_execute))
 
     # Noeuds ---------------------------------------------------------------------------------------
     logger.info("Writing vertices...")
-    cursor_in = connection_work.cursor(cursor_factory=DictCursor, name="cursor_in")
     create_nodes = """
         DROP TABLE IF EXISTS {0}_vertices_pgr;
         CREATE TABLE {0}_vertices_pgr(
@@ -165,34 +161,26 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             eout int,
             the_geom geometry(Point,4326)
     );""".format(ways_table_name)
-    logger.debug("SQL: {}".format(create_nodes))
-    cursor_out.execute(create_nodes)
+    database_out.execute_update(create_nodes)
 
     logger.info("Populating vertices")
     nd_query = f"SELECT id, geom FROM {input_schema}.nodes;"
-
-    logger.debug("SQL: {}".format(nd_query))
-    st_execute = time.time()
-    cursor_in.execute(nd_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
     # Insertion petit à petit -> plus performant
     # logger.info("SQL: Inserting or updating {} values in out db".format(cursor_in.rowcount))
     st_execute = time.time()
     index = 0
     batchsize = 10000
-    rows = cursor_in.fetchmany(batchsize)
+    generator = database_work.execute_select_fetch_multiple(nd_query, show_duration=True, batchsize=batchsize)
+    rows, count = next(generator, (None, None))
     while rows:
         values_str = ""
-        for row in rows:
-            values_str += "(%s, %s),"
-        values_str = values_str[:-1]
-
         # Tuple des valeurs à insérer
         values_tuple = ()
         for row in rows:
+            values_str += "(%s, %s),"
             values_tuple += (row['id'], row['geom'])
             index += 1
+        values_str = values_str[:-1]
 
         set_on_conflict = (
             "the_geom = excluded.the_geom"
@@ -204,18 +192,15 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             ON CONFLICT (id) DO UPDATE
               SET {};
         """.format(ways_table_name, values_str, set_on_conflict)
-        cursor_out.execute(sql_insert, values_tuple)
-        connection_out.commit()
-        rows = cursor_in.fetchmany(batchsize)
+        database_out.execute_update(sql_insert, values_tuple)
+        rows, _ = next(generator,(None, None))
 
 
     et_execute = time.time()
-    cursor_in.close()
     logger.info("Writing vertices Done. Elapsed time : %s seconds." %(et_execute - st_execute))
 
     # Ways -----------------------------------------------------------------------------------------
     # Colonnes à lire dans la base source (champs classiques + champs servant aux coûts)
-    cursor_in = connection_work.cursor(cursor_factory=DictCursor, name="cursor_in")
     attribute_columns = [
             'id',
             'geom as the_geom',
@@ -241,7 +226,6 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             'nombre_de_voies as nombre_de_voies',
             'insee_commune_gauche as insee_commune_gauche',
             'insee_commune_droite as insee_commune_droite',
-            'bande_cyclable as bande_cyclable',
             'itineraire_vert as itineraire_vert',
             'sens_de_circulation as sens_de_circulation',
             'reserve_aux_bus as reserve_aux_bus',
@@ -256,7 +240,12 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             'matieres_dangereuses_interdites as matieres_dangereuses_interdites',
             'cpx_gestionnaire as cpx_gestionnaire',
             'cpx_numero_route_europeenne as cpx_numero_route_europeenne',
-            'cpx_classement_administratif as cpx_classement_administratif'
+            'cpx_classement_administratif as cpx_classement_administratif',
+            'transport_exceptionnel as transport_exceptionnel',
+            'vla_par_defaut as vla_par_defaut',
+            'cout_penalites as cout_penalites',
+            'vehicule_leger_interdit as vehicule_leger_interdit',
+            'cout_vehicule_prioritaire as cout_vehicule_prioritaire'
         ]
     in_columns = attribute_columns.copy()
     for variable in costs["variables"]:
@@ -266,11 +255,8 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
 
     # Ecriture des ways
     sql_query = getQueryByTableAndBoundingBox(f'{input_schema}.edges', source['bbox'], in_columns)
-    logger.info("SQL: {}".format(sql_query))
-    st_execute = time.time()
-    cursor_in.execute(sql_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
+    batchsize = 10000
+    generator = database_work.execute_select_fetch_multiple(sql_query, show_duration=True, batchsize=batchsize)
 
     # Chaîne de n %s, pour l'insertion de données via psycopg
     single_value_str = "%s," * (len(attribute_columns) + 2 * len(costs["outputs"]))
@@ -279,24 +265,21 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
     # Insertion petit à petit -> plus performant
     # logger.info("SQL: Inserting or updating {} values in out db".format(cursor_in.rowcount))
     st_execute = time.time()
-    batchsize = 10000
     percent = 0
-    rows = cursor_in.fetchmany(batchsize)
+    rows, count = next(generator, (None, None))
     while rows:
-        percent += 1000000 / cursor_in.rowcount
+        percent += 1000000 / count
         # Chaîne permettant l'insertion de valeurs via psycopg
         values_str = ""
-        for row in rows:
-            values_str += "(" + single_value_str + "),"
-        values_str = values_str[:-1]
-
         # Tuple des valuers à insérer
         values_tuple = ()
         for row in rows:
+            values_str += "(" + single_value_str + "),"
             output_costs = output_costs_from_costs_config(costs, row)
             values_tuple += tuple(
                 row[ output_columns_name ] for output_columns_name in output_columns_names
             ) + output_costs
+        values_str = values_str[:-1]
 
         output_columns = "("
         for output_columns_name in output_columns_names:
@@ -320,12 +303,10 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
             ON CONFLICT (id) DO UPDATE
               SET {};
             """.format(ways_table_name, output_columns, values_str, set_on_conflict)
-        cursor_out.execute(sql_insert, values_tuple)
-        connection_out.commit()
-        rows = cursor_in.fetchmany(batchsize)
+        database_out.execute_update(sql_insert, values_tuple)
+        rows, _ = next(generator,(None, None))
 
     et_execute = time.time()
-    cursor_in.close();
     logger.info("Writing ways ended. Elapsed time : %s seconds." %(et_execute - st_execute))
 
     spacial_indices_query = """
@@ -335,60 +316,31 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
         CLUSTER {0}_vertices_pgr USING ways_vertices_geom_gist ;
         CREATE INDEX IF NOT EXISTS ways_importance_idx ON {0} USING btree (importance);
     """.format(ways_table_name)
-    logger.info("SQL: {}".format(spacial_indices_query))
-    st_execute = time.time()
-    cursor_out.execute(spacial_indices_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-    connection_out.commit()
+    database_out.execute_update(spacial_indices_query)
 
     turn_restrictions_indices_query = """
         CREATE INDEX IF NOT EXISTS turn_restrictions_id_key ON {0}.turn_restrictions USING btree (id);
         CREATE INDEX IF NOT EXISTS ways_id_key ON {1} USING btree (id);
         CREATE INDEX IF NOT EXISTS ways_vertices_pgr_id_key ON {1}_vertices_pgr USING btree (id);
     """.format(schema, ways_table_name)
-    logger.info("SQL: {}".format(turn_restrictions_indices_query))
-    st_execute = time.time()
-    cursor_out.execute(turn_restrictions_indices_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-    connection_out.commit()
+    database_out.execute_update(turn_restrictions_indices_query)
 
-    old_isolation_level = connection_out.isolation_level
-    connection_out.set_isolation_level(0)
 
     # VACCUM ANALYZE for ways
     vacuum_query = f"VACUUM ANALYZE {ways_table_name};"
-    logger.info("SQL: {}".format(vacuum_query))
-    st_execute = time.time()
-    cursor_out.execute(vacuum_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
+    database_out.execute_update(vacuum_query, isolation_level=0)
 
     # VACCUM ANALYZE for ways_vertices_pgr
     vacuum_query = f"VACUUM ANALYZE {ways_table_name}_vertices_pgr;"
-    logger.info("SQL: {}".format(vacuum_query))
-    st_execute = time.time()
-    cursor_out.execute(vacuum_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
+    database_out.execute_update(vacuum_query, isolation_level=0)
 
     # VACCUM ANALYZE for turn_restrictions
     vacuum_query = f"VACUUM ANALYZE {schema}.turn_restrictions;"
-    logger.info("SQL: {}".format(vacuum_query))
-    st_execute = time.time()
-    cursor_out.execute(vacuum_query)
-    et_execute = time.time()
-    logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-
-    connection_out.set_isolation_level(old_isolation_level)
-    connection_out.commit()
+    database_out.execute_update(vacuum_query, isolation_level=0)
 
-    cursor_out.close()
 
     # Nettoyage du graphe
     logger.info("Cleaning isolated clusters of less than 10 edges...")
-    cursor_isolated = connection_out.cursor()
 
     profile_names = set([ cost['profile'] for cost in source["costs"]])
     st_execute = time.time()
@@ -427,12 +379,10 @@ def pivot_to_pgr(source, cost_calculation_file_path, connection_work, connection
         WHERE {0}.ways.target = ANY(SELECT * from remove_nodes) OR {0}.ways.source = ANY(SELECT * from remove_nodes);
         """.format(schema, profile_name)
         logger.info("SQL: {}".format(clean_graph_query))
-        cursor_isolated.execute(clean_graph_query)
-        connection_out.commit()
+        database_out.execute_update(clean_graph_query)
 
     et_execute = time.time()
     logger.info("Execution ended. Elapsed time : %s seconds." %(et_execute - st_execute))
-    cursor_isolated.close()
 
     end_time = time.time()
     logger.info("Conversion from pivot to PGR ended. Elapsed time : %s seconds." %(end_time - start_time))

r2gg/cli.py

@@ -3,38 +3,30 @@
 """Main CLI entrypoint."""
 
 # Package
-from r2gg.__about__ import (
-    __author__,
-    __cli_usage__,
-    __summary__,
-    __title__,
-    __title_clean__,
-    __uri_homepage__,
-    __version__,
-)
-from r2gg._configure import configure, connect_working_db, disconnect_working_db
+from r2gg._configure import configure
 from r2gg._main import sql_convert, pgr_convert, osm_convert, osrm_convert, valhalla_convert, write_road2_config
+from r2gg._database import DatabaseManager
 
 # ############################################################################
 # ########## MAIN ################
 # ################################
 def sql2pivot():
     config, resource, db_configs, logger = configure()
-    connection = connect_working_db(config, db_configs, logger)
-    sql_convert(config, resource, db_configs, connection, logger)
-    disconnect_working_db(connection, logger)
+    database = DatabaseManager(db_configs[config["workingSpace"]["baseId"]], logger)
+    sql_convert(config, resource, db_configs, database, logger)
+    database.disconnect_working_db()
 
 def pivot2pgrouting():
     config, resource, db_configs, logger = configure()
-    connection = connect_working_db(config, db_configs, logger)
-    pgr_convert(config, resource, db_configs, connection, logger)
-    disconnect_working_db(connection, logger)
+    database = DatabaseManager(db_configs[config["workingSpace"]["baseId"]], logger)
+    pgr_convert(resource, db_configs, database, logger)
+    database.disconnect_working_db()
 
 def pivot2osm():
     config, resource, db_configs, logger = configure()
-    connection = connect_working_db(config, db_configs, logger)
-    osm_convert(config, resource, db_configs, connection, logger)
-    disconnect_working_db(connection, logger)
+    database = DatabaseManager(db_configs[config["workingSpace"]["baseId"]], logger)
+    osm_convert(config, resource, db_configs, database, logger)
+    database.disconnect_working_db()
 
 def osm2osrm():
     config, resource, _, logger = configure()
@@ -52,20 +44,14 @@ def main():
     """Main CLI entrypoint.
     """
     config, resource, db_configs, logger = configure()
-    connection = connect_working_db(config, db_configs, logger)
-    sql_convert(config, resource, db_configs, connection, logger)
-    if (resource['type'] in ['pgr', 'smartpgr']):
-        config, resource, db_configs, connection, logger = configure()
-        pgr_convert(config, resource, db_configs, connection, logger)
-        disconnect_working_db(connection, logger)
-    elif (resource['type'] == 'osrm'):
-        config, resource, db_configs, connection, logger = configure()
-        osm_convert(config, resource, db_configs,  connection, logger)
-        disconnect_working_db(connection, logger)
+    sql2pivot()
+    if resource['type'] in ['pgr', 'smartpgr']:
+        pivot2pgrouting()
+    elif resource['type'] == 'osrm':
+        pivot2osm()
         osrm_convert(config, resource, logger)
-    elif (resource['type'] == 'valhalla'):
-        config, resource, db_configs, connection, logger = configure()
-        osm_convert(config, resource, db_configs, connection, logger, True)
+    elif resource['type'] == 'valhalla':
+        pivot2osm()
         valhalla_convert(config, resource, logger)
     else:
         raise ValueError("Wrong resource type, should be in ['pgr',osrm','valhalla','smartpgr']")