IservETLLib 0.0.1__py3-none-any.whl

IservETLLib/DatabaseManager.py

@@ -0,0 +1,134 @@
+import json
+from sqlalchemy import create_engine, text, URL
+from sqlalchemy.exc import SQLAlchemyError
+
+class DBConfig:
+    def __init__(self, host, port, user, password, database):
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.database = database
+    
+    def __str__(self):
+        return f"DatabaseConfig(host={self.host}, port={self.port}, user={self.user}, database={self.database})"
+    
+    def __repr__(self):
+        return self.__str__()
+
+class DBManager:
+    def __init__(self, application_name, config_string, logger):
+        self.db_config = self.parse_db_config(config_string)
+        self.url_object = URL.create(
+            drivername="postgresql",
+            username=self.db_config.user,
+            password=self.db_config.password,
+            host=self.db_config.host,
+            database=self.db_config.database,
+        )
+        self.application_name = application_name
+        if logger:
+            self.logger = logger
+
+    def parse_db_config(self, config_string):
+        """Парсит строку конфигурации создает DatabaseConfig"""
+        config_dict = {}
+        
+        # Разделяем по точкам с запятой
+        pairs = config_string.split(';')
+        
+        for pair in pairs:
+            if ':' in pair:
+                key, value = pair.split(':', 1)  # split только по первому двоеточию
+                config_dict[key.strip()] = value.strip()
+        
+        # Создаем объект конфигурации
+        return DBConfig(
+            host=config_dict.get('host', ''),
+            port=int(config_dict.get('port', 0)),
+            user=config_dict.get('user', ''),
+            password=config_dict.get('password', ''),
+            database=config_dict.get('database', '')
+        )
+    
+    def query_database(self, connection_string, query, queryParams):
+        try:
+            engine = create_engine(connection_string, 
+                connect_args={
+                    "application_name": self.application_name
+                }
+            )
+            
+            with engine.connect() as conn:
+                # Выполнение запроса с параметрами
+                result = conn.execute(query, queryParams)
+                #print("[query_database]", query, queryParams)
+                # Получение всех результатов
+                items = result.fetchall()
+                
+                conn.commit()
+                return items
+                
+        except SQLAlchemyError as e:
+            error_msg = f"Database error: {e}"
+            self.logger.log(f'{error_msg} {query} {queryParams}')
+            return None
+            
+        finally:
+            # Гарантированное отключение от БД
+            if 'engine' in locals():
+                engine.dispose()
+
+
+    def of_users(sender_id):
+        query = text("""
+            SELECT * FROM core.of_users(
+                (SELECT to_jsonb(t) FROM core.sf_users(:sender_id) as t), 
+                '{"c_profiles":["user", "brigade"]}'::jsonb
+            );
+            """
+        )
+        queryParams = {
+            "sender_id": sender_id
+        }
+        return query, queryParams
+
+
+    def of_levels(sender_id):
+        query = text("""
+            SELECT * FROM core.of_levels(
+                (SELECT to_jsonb(t) FROM core.sf_users(:sender_id) as t), 
+                null
+            );
+            """
+        )
+        queryParams = {
+            "sender_id": sender_id
+        }
+        return query, queryParams
+
+    def cs_event_types():
+        query = text("""
+            SELECT id, c_name, c_const, b_disabled, b_default
+            FROM dbo.cs_event_types;
+            """
+        )
+        queryParams = {}
+        return query, queryParams
+    
+    def of_arm_cd_user_events(sender_id, d_date_start, d_date_end):
+        query = text("""
+            SELECT * FROM dbo.of_arm_cd_user_events(
+                (SELECT to_jsonb(t) FROM core.sf_users(:sender_id) as t), 
+                :json_params
+            );
+            """
+        )
+        queryParams = {
+            "sender_id": sender_id,
+            "json_params": json.dumps({
+                "d_date_start": d_date_start,
+                "d_date_end": d_date_end
+            })
+        }
+        return query, queryParams

IservETLLib/Helpers.py

@@ -0,0 +1,17 @@
+import time
+from datetime import datetime
+
+class Helpers:
+    def generate_route_number(base_number):
+        """
+        Генерирует номер маршрута в формате BASE-YYYY-MM-DD-LAST2DIGITS
+
+        Parameters
+        ----------
+        base_number : int
+            Организационное подразделение пользователя
+        """
+
+        now = datetime.now()
+        timestamp_ms = int(time.time() * 1000)
+        return f"{base_number}-{now.strftime('%Y-%m-%d')}-{str(timestamp_ms)[-2:]}"

IservETLLib/LogManager.py

@@ -0,0 +1,80 @@
+import os
+import uuid
+from datetime import datetime
+from enum import Enum
+from fluent import sender, event
+
+class FluentConf:
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
+    
+    
+class LogLevel(Enum):
+    INFO = 'INFO'
+    ERROR = 'ERROR'
+    CRITICAL = 'CRITICAL'
+
+class Logger:
+    def __init__(self, application_name, file):
+        self.file = file
+        self.application_name = application_name
+        self.fluent = None
+
+    def init_fluent(self, level: LogLevel, fluent: FluentConf):
+        if not fluent:
+            raise Exception('Конфиг пустой')
+            return
+        
+        sender.fluent = fluent
+        sender.setup(level, host=fluent.host, port=fluent.port)
+
+
+    def log(self, message):
+        self.__log(LogLevel.INFO, message)
+
+    def error(self, message, b_raise=True):
+        self.__log((LogLevel.CRITICAL if b_raise else LogLevel.ERROR), message)
+
+    def __log(self, level: LogLevel, message):
+        """
+        Parameters
+        ----------
+        level: LogLevel 
+            уровень INFO ERROR CRITICAL
+        message : str
+            входная строка
+
+        Raises
+        -------
+        Exception
+            Исключение при записи в лог-файл
+        """
+        datetime_string = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        message = f'{level.name} {datetime_string}: {message}'
+        print(message)
+
+        file = self.file
+        try:
+            # Создаем директорию, если она не существует
+            dir = os.path.dirname(file)
+            if dir:  # Если путь содержит директорию
+                os.makedirs(dir, exist_ok=True)
+
+            with open(file, 'a', encoding='utf-8') as f:
+                f.write(message + '\n')
+        except Exception as e:
+            print(f'Ошибка записи в лог-файл {file}: {e}')
+
+        if self.fluent:
+            event.Event(level.name, {
+                "id": f"etl.{self.application_name}.{uuid.uuid4()}",
+                "@timestamp": datetime_string,
+                "message": message
+            })
+
+        if level == LogLevel.CRITICAL:
+            raise Exception(message)
+        
+
+    

IservETLLib/__init__.py

@@ -0,0 +1,2 @@
+from .DatabaseManager import *
+from .LogManager import *

IservETLLib/index.py

@@ -0,0 +1,26 @@
+import pandas as pd
+from DatabaseManager import DBManager
+from LogManager import Logger, LogLevel, FluentConf
+from Helpers import Helpers
+
+
+CONNECTION_STRING = "host:192.168.17.46;port:5432;user:user;password:password;database:skr-dev-db"
+ETL = "import_calendar.ipynb"
+OUTPUT_DIR = "./files/output"
+
+__CONNECTION_STRING=CONNECTION_STRING if 'CONNECTION_STRING' in locals() else None
+__OUTPUT_DIR=OUTPUT_DIR if 'OUTPUT_DIR' in locals() else ''
+__ETL=ETL if 'ETL' in locals() else 'unknown etl'
+# __USR_ID=USR_ID if 'USR_ID' in locals() else 'null'
+# # __FILEPATHNAME=FILEPATHNAME if 'FILEPATHNAME' in locals() else ''
+# __FILEPATHNAME=os.path.join(OUTPUT_DIR.rsplit('/output', 1)[0], FILEPATHNAME) if 'FILEPATHNAME' in locals() else None #Используются на проде
+__LOG_FILE = __OUTPUT_DIR + '/output.log'
+
+
+if __name__ == "__main__":
+
+    logger = Logger(__ETL, __LOG_FILE)
+    dbm = DBManager(__ETL, __CONNECTION_STRING, logger)
+    # logger.init_fluent(LogLevel.INFO, FluentConf('10.199.236.8', 24224))
+    logger.log("asdas")
+    print(Helpers.generate_route_number('asdad'))

iservetllib-0.0.1.dist-info/METADATA

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: IservETLLib
+Version: 0.0.1
+Summary: Общий код для ETL jupiter notebooks
+Author-email: k-avtonomov <k-avtonomov@it-serv.ru>, a-kostyuhin <a-kostyuhin@it-serv.ru>
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# **Библиотека для ETL** # 
+
+Для формирования xlsx отчета из нескольких csv файлов.
+
+## Разработка ##
+
+    source venv/bin/activate
+
+## Установка ##
+
+    pip install IservETLLib
+
+## Использование ##
+
+python3 ./src/IservETLLib/index.py
+
+<!-- python3 OUTPUT_NAME="res.xlsx" HEADERS_PATH="" META_NAME="excel.meta" SEP=;  "Опросный лист ТУ_2025-07-28_14-28-44.csv" "Опросный лист УСПД_2025-07-28_14-33-04.csv" -->
+
+python3 -m build
+python3 -m twine upload --repository pypi dist/*

iservetllib-0.0.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+IservETLLib/DatabaseManager.py,sha256=kj8UW8vBPe8t5jpIRShRCPVjoNEKS_VSF_po6CGU5oI,4464
+IservETLLib/Helpers.py,sha256=7cYtn6Gaxx7lcJQ--xnf6QR0JlRcpTGnA0PMGY9Io60,549
+IservETLLib/LogManager.py,sha256=APaumC7EsRsNYwr4_YCm-ndEpv3_ZLtjZHINoC_oJvw,2288
+IservETLLib/__init__.py,sha256=5Jya_pCZ0MgDy39AaCuze13pkpBc3oWVFhCE7pcgcwk,56
+IservETLLib/index.py,sha256=uVNPJyrhTMpG_odz9Z4sJ-S8LhXSqcPqcty1DhArgcU,1095
+iservetllib-0.0.1.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+iservetllib-0.0.1.dist-info/METADATA,sha256=onaVHd3cSmfncCyejVVW-hB8JOyEo79gOoLijTfjQfI,1018
+iservetllib-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+iservetllib-0.0.1.dist-info/top_level.txt,sha256=KWjGCYc5VEWu2FPx4yNB4SozwwJUuPDSuDrJ5v6JLR8,12
+iservetllib-0.0.1.dist-info/RECORD,,

iservetllib-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

iservetllib-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+IservETLLib