quickinteg 2.5.0__py3-none-any.whl

quickinteg/__about__.py

@@ -0,0 +1,34 @@
+#! python3  # noqa: E265
+
+"""
+Metadata bout the package to easily retrieve informations about it.
+See: https://packaging.python.org/guides/single-sourcing-package-version/
+"""
+
+from datetime import date
+
+__all__ = [
+    "__author__",
+    "__copyright__",
+    "__email__",
+    "__license__",
+    "__summary__",
+    "__title__",
+    "__uri__",
+    "__version__",
+]
+
+__author__ = "Valérian LEBERT"
+__copyright__ = f"2019 - {date.today().year}, {__author__}"
+__email__ = ""
+__executable_name__ = ""
+__license__ = "GPL"
+__summary__ = "Scripts QGIS Processing pour l'intégration rapide des données."
+__title__ = "QuickInteg"
+__title_clean__ = "".join(e for e in __title__ if e.isalnum())
+__uri__ = "https://gitlab.com/vlebert/quickinteg/"
+
+__version__ = "2.5.0"
+__version_info__ = tuple(
+    [int(num) if num.isdigit() else num for num in __version__.replace("-", ".", 1).split(".")]
+)

quickinteg/cfg_reader.py

@@ -0,0 +1,104 @@
+#! python3  # noqa: E265
+
+################################################################################
+# This file is part of quickinteg.
+
+# quickinteg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# quickinteg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with quickinteg.  If not, see <https://www.gnu.org/licenses/>.
+################################################################################
+
+# ############################################################################
+# ########## Imports ###############
+# ##################################
+# standard library
+import ast
+import configparser
+import logging
+import os
+from pathlib import Path
+
+# package
+from . import mcd_check, processing_logging, spatialiteio
+
+# ############################################################################
+# ########## Globals ###############
+# ##################################
+
+module_logger = logging.getLogger(__name__)
+module_logger.setLevel(logging.DEBUG)
+
+
+# ############################################################################
+# ########## Functions #############
+# ##################################
+
+
+def log(message: str, logLevel="info"):
+    processing_logging.log(str(message), module_logger, logLevel)
+
+
+def parse_config_file(config_file_path: str) -> dict:
+    raw_config = configparser.ConfigParser()
+    raw_config.read(config_file_path)
+
+    config = dict()
+    l_params = ["table_def", "mcd_check", "sql", "export_table"]
+    log("Lecture du fichier de configuration %s" % (config_file_path))
+    for param in l_params:
+        try:
+            config[param] = ast.literal_eval(raw_config["template"][param])
+            log("%s: %s" % (param, config[param]))
+        except KeyError:
+            log("Pas de valeur %s dans la config" % (param))
+        except ValueError:  # si pas de syntaxe python (ex, guillements pour string)
+            config[param] = raw_config["template"][param]
+        except SyntaxError:  # si pas de syntaxe python (ex, guillements pour string)
+            config[param] = raw_config["template"][param]
+
+    log("Configuration : \n%s" % (config))
+    return config
+
+
+def create_template_from_cfg(config_file_path):
+    config = parse_config_file(config_file_path)
+    os.chdir(Path(config_file_path).parent)
+
+    sql_tables_def_folder = config["table_def"]
+
+    base_spl_filename = Path(sql_tables_def_folder).name
+    base_spl_path = Path(base_spl_filename + ".sqlite")
+    # On crée une base sqlite3 avec extension spatialite
+    if base_spl_path.exists():
+        raise ValueError("Le fichier %s existe déjà" % (base_spl_path))
+    else:
+        spatialiteio.create_spatialite_db(base_spl_path)
+
+    # On exécute les différents scripts SQL sur la base spatialite
+
+    # MCD CHECK
+    spatialiteio.execute_file_to_spl(sql_tables_def_folder, base_spl_path)
+
+    # Configuration de contrôle
+    if "mcd_check" in set(config.keys()):
+        catalog_file = str(Path(config["mcd_check"]["catalogue"]))
+        config_file = str(Path(config["mcd_check"]["config"]))
+        sql_code = mcd_check.translate_config_to_sql(catalog_file, config_file)
+        spatialiteio.execute_query_to_spl(sql_code, base_spl_path)
+
+    # SQL
+    if "sql" in set(config.keys()):
+        for sql in config["sql"]:
+            spatialiteio.execute_file_to_spl(str(Path(sql)), base_spl_path)
+
+    if "export_table" in set(config.keys()):
+        spatialiteio.check_preset_tables(base_spl_path, config["export_table"])

quickinteg/cli.py

@@ -0,0 +1,144 @@
+#! python3  # noqa: E265
+
+################################################################################
+# This file is part of quickinteg.
+
+# quickinteg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# quickinteg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with quickinteg.  If not, see <https://www.gnu.org/licenses/>.
+################################################################################
+
+
+import argparse
+import logging
+from pathlib import Path
+
+from . import cfg_reader, processing_logging, rop_grace, spatialiteio
+
+
+def quickinteg_create():
+    """Permet de créer un alias CLI pour appeler cfg_reader.create_template_from_cfg.
+    Alias : qi-create
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("config_file", help="Chemin vers le fichier de configuration de template")
+    args = parser.parse_args()
+    processing_logging.add_console_handler(logging.getLogger(""))
+    cfg_reader.create_template_from_cfg(args.config_file)
+
+
+def quickinteg_delete():
+    """Alias CLI pour faire un delete à partir du import_prefix"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("db", help="Base spatialite dans laquelle supprimer les lignes")
+    parser.add_argument("import_prefix", help="Valeur de l'import prefix")
+    args = parser.parse_args()
+    processing_logging.add_console_handler(logging.getLogger(""))
+    spatialiteio.delete_rows(args.db, args.import_prefix)
+
+
+# TODO : add log to text
+def quickinteg_import():
+    """Permet de créer un alias CLI pour appeler spatialiteio.import_folder_to_spl et
+    spatialiteio.export_preset_tables.
+    Alias : qi-import
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("source", help="Chemin vers le repertoir à intégrer")
+    parser.add_argument("-c", "--no-csv", help="Exclure les fichiers CSV", action="store_true")
+    parser.add_argument(
+        "-d",
+        "--destination",
+        help="Chemin vers une base de donnée existante (mode APPEND).",
+        default=None,
+    )
+    parser.add_argument(
+        "-p",
+        "--prefix",
+        help="Préfixe d'importation ajouté sur chaque table.",
+        default="",
+    )
+    parser.add_argument("-e", "--export", help="Exporter les vues de contrôle", action="store_true")
+    parser.add_argument(
+        "-r",
+        "--recursive",
+        help="Mode récursif (parcours les sous-dossiers).",
+        action="store_true",
+    )
+    args = parser.parse_args()
+    processing_logging.add_console_handler(logging.getLogger(""))
+
+    if args.destination:
+        log_file = Path(args.destination).parent / "log_integ.txt"
+    else:
+        log_file = Path(args.source) / "log_integ.txt"
+    processing_logging.add_file_handler(logging.getLogger(""), log_file, logging.DEBUG)
+
+    spatialiteio.import_folder_to_spl(
+        args.source,
+        ogr_target=args.destination,
+        prefix=args.prefix,
+        include_csv=(not args.no_csv),
+        ignore_grace_list=True,
+        recursive=args.recursive,
+    )
+    if args.export:
+        spatialiteio.export_preset_tables(args.destination)
+
+
+def quickinteg_rop():
+    """Permet de créer un alias CLI générer les routes optiques.
+    Alias : qi-rop
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("db_file", help="Base de donnée Quickinteg")
+    parser.add_argument(
+        "-v", "--version", help="Version GRACE THD (2 ou 3, 3 par défaut)", default=3
+    )
+    parser.add_argument(
+        "-m",
+        "--mode",
+        help="Mode to use (table or view, table by default)",
+        default="table",
+    )
+    args = parser.parse_args()
+    processing_logging.add_console_handler(logging.getLogger(""))
+    if int(args.version) == 3:
+        rop_grace.param_st_id = "lc_codeext"
+        rop_grace.param_cb_id = "cb_codeext"
+        rop_grace.param_bp_id = "bp_codeext"
+        rop_grace.param_ti_id = "ti_codeext"
+        rop_grace.param_lg_cable = "cb_lgreel"
+        rop_grace.grace_version = 3
+        rop_grace.mode = args.mode
+        rop_grace.generate_ropt_full(args.db_file, n_iteration=15)
+    elif int(args.version) == 2:
+        rop_grace.param_st_id = "st_codeext"
+        rop_grace.param_cb_id = "cb_codeext"
+        rop_grace.param_bp_id = "bp_codeext"
+        rop_grace.param_ti_id = "ti_codeext"
+        rop_grace.param_lg_cable = "cb_lgreel"
+        rop_grace.grace_version = 2
+        rop_grace.mode = args.mode
+        rop_grace.generate_ropt_full(args.db_file, n_iteration=15)
+    else:
+        raise ValueError("Valeur incorrecte pour -v / --version")
+
+
+def quickinteg_formatrop():
+    """Permet de créer un alias CLI formater les routes optique au format excel.
+    Alias : qi-formatrop
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("rop_file", help="Fichier de ROP")
+    args = parser.parse_args()
+    rop_grace.format_ropt(args.rop_file)

quickinteg/excel_shortcut.py

@@ -0,0 +1,163 @@
+#! python3  # noqa: E265
+
+################################################################################
+# This file is part of quickinteg.
+
+# quickinteg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# quickinteg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with quickinteg.  If not, see <https://www.gnu.org/licenses/>.
+################################################################################
+
+# ############################################################################
+# ########## Libraries #############
+# ##################################
+
+# standard library
+import logging
+
+# 3rd party - try embedded first (for QGIS), fallback to normal import (for CLI)
+try:
+    from .external import openpyxl
+except ImportError:
+    import openpyxl
+
+# package
+from . import processing_logging
+
+# ############################################################################
+# ########## Globals ###############
+# ##################################
+
+module_logger = logging.getLogger(__name__)
+module_logger.setLevel(logging.DEBUG)
+
+
+# ############################################################################
+# ########## Functions #############
+# ##################################
+
+
+def log(message: str, logLevel: str = "info"):
+    """Map log message on different logging pipes. Mainly: QGIS Processing log system.
+
+    Args:
+        message (str): [description]
+        logLevel (str, optional): [description]. Defaults to "info".
+    """
+    processing_logging.log(message, module_logger, logLevel)
+
+
+def read_sheet_to_dict(
+    excel_file_path, list_necessary_columns, primary_key=None, excel_sheet_name=None
+):
+    """
+    Lecture d'un fichier excel vert un dict de dict.
+    Chaque ligne est un dict avec comme clés les en-têtes de colones.
+    Le fichier est un dict de lignes avec comme clé la valeur \
+    dans la colonne "primary_key".
+    Si primary_key == None, on utilise le numéro de ligne
+    ex :
+
+    le tableau
+
+    Primary_key     |   colonne1    |   colonne2
+    key1            |   value1      |   value2
+    key2            |   value3      |   value4
+
+    donnera en sortie un dict
+    {
+        key1 {
+            Primary_key : key1,
+            colonne1 : value1,
+            colonne2 : value2
+            },
+        key2 {
+            Primary_key : key2,
+            colonne1 : value3,
+            colonne2 : value4
+            }
+        }
+    """
+    wb = openpyxl.load_workbook(excel_file_path, data_only=True)
+    sheet = wb[excel_sheet_name] if excel_sheet_name else wb.active
+
+    rows = sheet.iter_rows(values_only=True)
+    headers = next(rows)  # Get the first row with column names
+    header_set = set(headers)
+
+    necessary_columns_set = set(list_necessary_columns)
+    if not necessary_columns_set.issubset(header_set):
+        missing_columns = necessary_columns_set - header_set
+        raise ValueError(
+            f"The necessary columns {missing_columns} were not found in the Excel sheet"
+        )
+
+    header_index = {header: headers.index(header) for header in list_necessary_columns}
+
+    dict_output = {}
+    for i, row in enumerate(rows, start=1):  # start=1 because Excel is 1-indexed
+        row_data = {header: row[header_index[header]] for header in list_necessary_columns}
+        key = row_data.get(primary_key) if primary_key else i
+        dict_output[key] = row_data
+
+    return dict_output
+
+
+def read_all_sheet_to_dict(excel_file_path, list_necessary_columns, primary_key):
+    dict_output = dict()
+    wb = openpyxl.load_workbook(excel_file_path, data_only=True)
+    for sheet_name in wb.sheetnames:
+        dict_output.update(
+            read_sheet_to_dict(
+                excel_file_path,
+                list_necessary_columns,
+                primary_key,
+                excel_sheet_name=sheet_name,
+            )
+        )
+    return dict_output
+
+
+def read_sheet_to_tab(excel_file_path, excel_sheet_name=None):
+    """Lecture d'un fichier excel et retour un dict de deux listes :
+    une liste header
+    une double liste data"""
+    # Ouverture du xlsx
+    wb = openpyxl.load_workbook(excel_file_path, data_only=True)
+
+    # Si pas de sheet_name passé en paramètre, on ouvre le premier onglet
+    if excel_sheet_name is None:
+        excel_sheet = wb[wb.sheetnames[0]]
+    # Sinon on ouvre l'onglet passé en paramètre
+    else:
+        excel_sheet = wb[excel_sheet_name]
+
+    # Stockage des valeurs dans un tableau
+    header = []
+    data = []
+    for i, row in enumerate(excel_sheet.iter_rows(values_only=True)):
+        if i == 0:
+            header.extend(row)
+        else:
+            data.append(list(row))
+    return {"header": header, "data": data}
+
+
+# ############################################################################
+# ########## Main ##################
+# ##################################
+
+# for compatibility after simplifying name
+read_excel_sheet_to_dict = read_sheet_to_dict
+
+# for compatibility after simplifying name
+read_excel_sheet_to_tab = read_sheet_to_tab