utilskit 0.1.0__tar.gz

utilskit-0.1.0/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: utilskit
+Version: 0.1.0
+Summary: description
+Author: Kimyh
+Author-email: kim_yh663927@naver.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: matplotlib==3.10.3
+Requires-Dist: numpy==2.2.6
+Requires-Dist: pandas==2.3.1
+Requires-Dist: PyMySQL==1.1.1
+Requires-Dist: SQLAlchemy==2.0.41
+Requires-Dist: tqdm==4.67.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+baseline

utilskit-0.1.0/README.md

@@ -0,0 +1 @@
+baseline

utilskit-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

utilskit-0.1.0/setup.py

@@ -0,0 +1,33 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="utilskit",  # 패키지 이름 (pip install 시 사용될 이름)
+    version="0.1.0",    # 버전
+    packages=find_packages(),  # textbasic 폴더 내 모든 패키지 포함
+    include_package_data=True,  # 이 설정을 통해 패키지 내 데이터 파일을 포함시킬 수 있음
+    package_data={
+    },
+    install_requires=[ # 패키지 설치 시 같이 설치되도록 설정
+        "matplotlib==3.10.3",
+        "numpy==2.2.6",
+        "pandas==2.3.1",
+        "PyMySQL==1.1.1",
+        "SQLAlchemy==2.0.41",
+        "tqdm==4.67.1"
+    ],
+    # install_requires=[
+	#    "pandas>=1.3.0,<2.0.0",  # 버전 범위 설정 방법
+	# ],
+    author="Kimyh",
+    author_email="kim_yh663927@naver.com",
+    description="description",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    # url="https://github.com/Kim-YoonHyun/my_package",  # 깃허브 주소 등
+    classifiers=[   # 패키지의 Meta 데이터
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",   # 제약없는 가장 자유로운 라이센스(MIT 대학에서 이름이 유래했을뿐 MIT 가 관리하는 라이센스는 아님)
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.10",  # 최소 지원할 파이썬 버전
+)

utilskit-0.1.0/utilskit/__init__.py

@@ -0,0 +1,7 @@
+from . import classificationutils
+from . import dataframeutils
+from . import dbutils
+from . import logutils
+from . import plotutils
+from . import timeutils
+from . import utils

utilskit-0.1.0/utilskit/classificationutils.py

@@ -0,0 +1,143 @@
+'''
+pip install pandas
+'''
+import sys
+import pandas as pd
+import numpy as np
+
+
+def get_max_2nd_n_reliability(pred):
+    pred_min = np.expand_dims(np.min(pred, axis=1), axis=1)
+    pred = pred - pred_min
+    pred_max = np.expand_dims(np.max(pred, axis=1), axis=1)
+    pred = pred/pred_max
+
+    # 1순위 예측값 없애기
+    pred = np.where(pred == 1, -100, pred)
+
+    # 2순위 예측
+    max_2nd_index = np.argmax(pred, axis=1)
+
+    # 신뢰도 구하기
+    pred_reliability = (1 - np.max(pred, axis=1))*100
+    return max_2nd_index, pred_reliability
+
+
+def matrix2confusion(matrix, uni_label_list, round_num=4, show_percentage=True):
+    whole_sum = np.sum(matrix)
+    true_sum_list = np.sum(matrix, axis=-1).tolist()
+    pred_sum_list = np.sum(matrix, axis=-2).tolist()
+    
+    # make matrix
+    if show_percentage:
+        per_num = 100
+    else:
+        per_num = 1
+    correct_sum = 0
+    for i in range(len(matrix)):
+        correct_count = matrix[i][i]
+        correct_sum += correct_count
+        pred_sum = pred_sum_list[i]
+        true_sum = true_sum_list[i]
+        
+        # precision
+        try:
+            precision = correct_count / pred_sum
+            precision = np.round(precision, round_num) * per_num
+        except ZeroDivisionError:
+            precision = None
+            
+        # recall
+        try:
+            recall = correct_count / true_sum
+            recall = np.round(recall, round_num) * per_num
+        except ZeroDivisionError:
+            recall = None
+            
+        # f1_score
+        try:
+            f1_score = 2*precision*recall / (precision + recall)
+            f1_score = np.round(f1_score, round_num)
+        except TypeError:
+            f1_score = None
+        
+        matrix[i].extend([None, precision, recall, f1_score, true_sum])
+        
+    whole_accuracy = correct_sum / whole_sum
+    whole_accuracy = np.round(whole_accuracy, round_num) * per_num
+    
+    # index & column
+    index_list = uni_label_list.copy()
+    index_list.append('count')
+    column_list = uni_label_list.copy()
+    column_list.extend(['accuracy', 'precision', 'recall', 'f1 score', 'count'])
+    
+    # count 추가
+    pred_count = pred_sum_list + [None]*(len(column_list) - len(index_list))
+    matrix.append(pred_count)
+    
+    # confusion matrix
+    confusion_matrix = pd.DataFrame(matrix, index=index_list, columns=column_list)
+    confusion_matrix['accuracy'][0] = whole_accuracy
+    
+    return confusion_matrix
+
+
+def make_confusion_matrix(mode, true_list, pred_list, ignore_idx=None, round_num=4, label2id_dict=None, id2label_dict=None, show_percentage=True):
+    if mode == 'label2id':
+        uni_label_list = list(label2id_dict.keys())
+    elif mode == 'id2label':
+        uni_label_list = list(id2label_dict.values())
+        
+    # matrix
+    matrix = []
+    for i in range(len(uni_label_list)):
+        matrix.append([])
+        for _ in range(len(uni_label_list)):
+            matrix[i].append(0)
+
+    # count
+    if mode == 'label2id':
+        for t, p in zip(true_list, pred_list):
+            t_i = label2id_dict[t]
+            p_i = label2id_dict[p]
+            matrix[t_i][p_i] += 1
+            
+    elif mode == 'id2label':
+        for t_i, p_i in zip(true_list, pred_list):
+            # padding 등의 idx 는 무시 
+            if (t_i is not None) and (t_i == ignore_idx):
+                continue
+            t_i = int(t_i)
+            p_i = int(p_i)
+            matrix[t_i][p_i] += 1
+            
+    confusion_matrix = matrix2confusion(
+        matrix=matrix, 
+        uni_label_list=uni_label_list, 
+        round_num=round_num, 
+        show_percentage=show_percentage
+    )
+    
+    return confusion_matrix
+
+
+def reset_confusion_matrix(confusion_matrix, new_label_list, round_num=4, show_percentage=True):
+    try:
+        matrix_df = confusion_matrix[new_label_list]
+    except KeyError:
+        print('예측 결과에 존재하지 않는 라벨명을 입력하였습니다.')
+        sys.exit()
+        
+    matrix_df = matrix_df.T[new_label_list]
+    matrix_df = matrix_df.T
+    matrix = matrix_df.values.tolist()
+    
+    new_confusion_matrix = matrix2confusion(
+        matrix=matrix, 
+        uni_label_list=new_label_list,
+        round_num=round_num,
+        show_percentage=show_percentage
+    )
+    
+    return new_confusion_matrix

utilskit-0.1.0/utilskit/dataframeutils.py

@@ -0,0 +1,273 @@
+import sys
+import os
+from datetime import datetime, timedelta
+import numpy as np
+import pandas as pd
+import csv
+import warnings
+warnings.filterwarnings('ignore')
+
+from utilskit import utils as u
+
+
+def read_df(path):
+    extention = path.split('.')[-1]
+    if extention in ['csv', 'CSV']:
+        switch = 'csv'
+    elif extention in ['xlsx', 'xls']:
+        switch = 'excel'
+    elif extention in ['txt']:
+        switch = 'txt'
+    else:
+        raise ValueError(f'{extention}은(는) 잘못되거나 지정되지 않은 확장자입니다.')
+    
+    if switch == 'csv':
+        encoding = 'utf-8-sig'
+        while True:
+            try:
+                data_df = pd.read_csv(path, encoding=encoding)
+                break
+            except UnicodeDecodeError:
+                encoding = 'cp949'
+            except pd.errors.ParserError:
+                f = open(path, encoding=encoding)
+                reader = csv.reader(f)
+                csv_list = []
+                for line in reader:
+                    if len(line) != 38:
+                        pass
+                    csv_list.append(line)
+                f.close()
+                data_df = pd.DataFrame(csv_list)
+                data_df.columns = data_df.iloc[0].to_list()
+                data_df = data_df.drop(index=data_df.index[0])	# 0번째 행을 지움
+                break
+    if switch == 'excel':
+        data_df = pd.read_excel(path)
+    if switch == 'txt':
+        line_list = []
+        with open(path, 'r', encoding='utf-8-sig') as f:
+            for line in f.readlines():
+                line = line.replace('\n', '')
+                line_list.append(line)
+        data_df = pd.DataFrame(line_list, columns=['string'])
+    return data_df
+
+
+def utc2kor(df, time_column='time'):
+    if df.empty:
+        return df
+    df[time_column] = df[time_column].astype('str')
+    df[time_column] = df[time_column].apply(lambda x: x.replace('T', ' '))
+    df[time_column] = df[time_column].apply(lambda x: x.replace('Z', ''))
+    
+    # UTC 시간을 한국 시간으로 (+9 시간)
+    df[time_column] = df[time_column].apply(lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
+    df[time_column] = df[time_column].apply(lambda x: x + timedelta(hours=9))
+    df[time_column] = df[time_column].astype('str')
+
+    df = df.sort_values(by=time_column,ascending=True)
+    
+    return df
+
+
+# def dataframe_preprocessor(df, 
+#                            max_dict=None, min_dict=None, 
+#                            nan_drop_column=None, 
+#                            do_nan_fill_whole=False
+#     ):
+#     # ========================================================================
+#     # # 최대값 초과 --> 이상치 --> 결측치
+#     # if max_dict is not None:
+#     #     df = maxadnormal2nan(df=df, max_dict=max_dict)
+        
+#     # # 최소값 미만 --> 이상치 --> 결측치
+#     # if min_dict is not None:
+#     #     df = minadnormal2nan(df=df, min_dict=min_dict)
+
+#     # ========================================================================
+#     # 특정 컬럼 기준 NaN 제거
+#     # if nan_drop_column is not None:
+#     #     df = drop_nan(
+#     #         df=df, 
+#     #         base_column=nan_drop_column
+#     #     )
+        
+#     # ========================================================================
+#     # 전후값 채우기
+#     if do_nan_fill_whole:
+#         df = df.fillna(method='ffill')
+#         df = df.fillna(method='bfill')
+    
+#     return df
+
+
+def adnormal2nan(df, stan_col, max_value=None, min_value=None):
+    if max_value is not None:
+        df[stan_col][df[stan_col] > max_value] = np.nan
+    if min_value is not None:
+        df[stan_col][df[stan_col] < min_value] = np.nan
+    return df
+
+
+def time_filling(df, start, end, time_column='time'):
+    if df.empty:
+        return df
+
+    time_range = pd.date_range(start=start, end=end, freq='S')
+    time_range_df = pd.DataFrame(time_range, columns=[time_column])
+    time_range_df = time_range_df.astype('str')
+
+    # 합치기
+    df = pd.merge(df, time_range_df, how='right')
+    return df
+
+
+def drop_nan(df, stan_col):
+    try:
+        df = df.dropna(subset=[stan_col])
+    except KeyError:
+        pass
+    return df
+
+
+def isdfvalid(df, valid_column_list):
+    # 유효 컬럼 존재 여부 확인
+    try:
+        _ = df[valid_column_list]
+        return True
+    except KeyError:
+        return False
+    
+    
+def local_nan_correction(df, stan_col, nan_repeat=5):
+    '''
+    nan_repeat 에 지정한 수치만큼 반복되는 결측치 구간을
+    앞뒤값 채우기로 보정하는 함수
+    '''
+    stan_ary = df[stan_col].values
+    nan_start_idx_list, nan_end_idx_list = u.identify_stan_repeat_section(
+        ary=stan_ary,
+        stan_value='nan',
+        stan_repeat=nan_repeat,
+        mode='below',
+        reverse=False
+    )
+    for nan_si, nan_ei in zip(nan_start_idx_list, nan_end_idx_list):
+        df.loc[nan_si-1:nan_ei, stan_col] = df.loc[nan_si-1:nan_ei, stan_col].fillna(method='ffill')
+        df.loc[nan_si:nan_ei+1, stan_col] = df.loc[nan_si:nan_ei+1, stan_col].fillna(method='bfill')
+    
+    return df
+
+
+def pin_nan_correction(df, stan_col, max_diff=0.1, nan_repeat=3):
+    '''
+    이상치 범위에 속하지 않지만 
+    데이터 흐름상 이상치로 볼 필요가 있는 국소 범위의 값들을 결측치로 변경하는 함수
+    
+    예시: 20, 20, 20, 20, [  1], 20, 20, 20, 1, 1, 2, 1
+    결과: 20, 20, 20, 20, [NaN], 20, 20, 20, 1, 1, 2, 1
+    '''
+    
+    # 기준 컬럼 데이터 추출
+    stan_ary = df[stan_col].values
+    
+    # 현재 값에서 이전값을 뺀 데이터 ary 를 생성
+    stan_1_list = stan_ary.tolist()
+    stan_1_list.insert(0, stan_ary[0])
+    stan_1_ary = np.array(stan_1_list)[:-1]
+    diff_ary = np.round(stan_ary - stan_1_ary, 4)
+    diff_ary = np.array(list(map(abs, diff_ary)))
+    
+    #==
+    # print()
+    idx_list = []
+    for idx, diff in enumerate(diff_ary):
+
+        # 앞뒤 차이값이 최대 차이값 보다 작은 경우
+        if diff < max_diff:
+            continue
+
+        # idx 위치 이전 10개 데이터에 대한 평균
+        before_aver = np.average(stan_ary[idx-10:idx])
+
+        # idx 위치 이후 10개 데이터에 대한 평균
+        after_aver = np.average(stan_ary[idx+1:idx+11])
+        
+        # 구간 내 nan 이 존재하는 경우 앞뒤 평균을 동일시
+        if str(before_aver) == 'nan':
+            before_aver = after_aver
+        if str(after_aver) == 'nan':
+            after_aver = before_aver
+        
+        # 앞뒤 평균값 간의 차이값 절대값 계산
+        aver_diff = abs(after_aver - before_aver)
+        
+        # 바로 앞 뒤의 차이값과 평균값 간 차이값의 차이값 p 계산
+        p = np.round(diff - aver_diff, 4)
+        
+        # p 가 최대 차이값 보다 큰 경우 이상치로 판단
+        if p > max_diff:
+            idx_list.append(idx)
+            
+        # print(f'{idx:5d}, {before_aver:.2f}, {diff:.2f}, {after_aver:.2f}, {aver_diff:.2f}')
+        # print(p)
+    # print(idx_list)
+    del idx
+    # idx_list = [0, 1, 11094, 11095, 12894, 12895, 12896, 12897, 35710, 35711]
+    # for i in idx_list:
+    #     print(stan_ary[i])
+    # del i
+    #==
+    
+    # print('-----------------------------------------')
+    # print(np.round(stan_ary[0:10], 4).tolist())
+    # print(np.round(stan_1_ary[0:10], 4).tolist())
+    # print(np.round(diff_ary[0:10], 4).tolist())
+    
+    # print('-----------------------------------------')
+    # print(np.round(stan_ary[11090:11100], 4).tolist())
+    # print(np.round(stan_1_ary[11090:11100], 4).tolist())
+    # print(np.round(diff_ary[11090:11100], 4).tolist())
+    
+    # print('-----------------------------------------')
+    # print(np.round(stan_ary[12890:12900], 4).tolist())
+    # print(np.round(stan_1_ary[12890:12900], 4).tolist())
+    # print(np.round(diff_ary[12890:12900], 4).tolist())
+    
+    # print('-----------------------------------------')
+    # print(np.round(stan_ary[17945:17955], 4).tolist())
+    # print(np.round(stan_1_ary[17945:17955], 4).tolist())
+    # print(np.round(diff_ary[17945:17955], 4).tolist())
+    
+    # print('-----------------------------------------')
+    # print(np.round(stan_ary[-10:], 4).tolist())
+    # print(np.round(stan_1_ary[-10:], 4).tolist())
+    # print(np.round(diff_ary[-10:], 4).tolist())
+    
+    # pin idx 가 존재하는 경우 해당 범위를 nan 으로 대체
+    temp_ary = stan_ary.copy()
+    if len(idx_list) > 0:
+        for idx in idx_list:
+            if idx < 3:
+                temp_ary[:idx+3] = np.nan
+            else:
+                temp_ary[idx-3:idx+3] = np.nan
+            
+    # nan 의 위치 구하기
+    for_fill_start_idx_list, for_fill_end_idx_list = u.identify_stan_repeat_section(
+        ary=temp_ary, 
+        stan_value='nan',
+        stan_repeat=nan_repeat, 
+        mode='below', 
+        reverse=False
+    )
+    
+    
+    # 해당 부분을 NaN 값으로 변환
+    for fsi, fei in zip(for_fill_start_idx_list, for_fill_end_idx_list):
+        df.loc[fsi:fei, stan_col] = np.nan
+        df.loc[fsi-1:fei, stan_col] = df.loc[fsi-1:fei, stan_col].fillna(method='ffill')
+        df.loc[fsi:fei+1, stan_col] = df.loc[fsi:fei+1, stan_col].fillna(method='bfill')
+        
+    return df

utilskit-0.1.0/utilskit/dbutils.py

@@ -0,0 +1,159 @@
+# DB
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+
+# def get_info():
+#     db_host = '192.168.0.85'
+#     db_port = 3306
+#     db_user = 'theimc'
+#     db_passward = 'theimc#10!'
+#     db_name = 'BUSMONITORING'
+#     charset = 'utf8mb4'
+#     if_exists = 'append'
+#     autocommit = True
+#     return (db_host, db_port, db_user, db_passward, db_name, charset, if_exists, autocommit)
+
+
+# def get_info_p5000():
+#     db_host = "59.25.131.135"
+#     db_port = 3306
+#     db_user = "ai_m"
+#     db_password = "temp"
+#     db_name = "bus"
+#     charset = 'utf8mb4'
+#     if_exists = 'append'
+#     autocommit = True
+#     return (db_host, db_port, db_user, db_password, db_name, charset, if_exists ,autocommit)
+
+
+def db_connect(host, user, port, passward, name, charset='utf8mb4', if_exists='append', autocommit=True):
+    # db_host = db_info_dict['host']
+    # db_port = db_info_dict['port']
+    # db_user = db_info_dict['user']
+    # db_passward = db_info_dict['passward']
+    # db_name = db_info_dict['name']
+    # charset = db_info_dict['charset']
+    # if_exists = db_info_dict['if_exists']
+    # autocommit = db_info_dict['autocommit']
+    conn = pymysql.connect(
+        host=host, 
+        user=user, 
+        port=port, 
+        password=passward, 
+        db=name,
+        charset=charset,
+        autocommit=autocommit
+    )
+    return conn
+
+
+def select_db(conn, query, where=None):
+    cursor = conn.cursor()
+    query = query
+    cursor.execute(query)
+    info = cursor.fetchall()
+    cursor.close()
+    return info
+
+
+def delete_db(db_info_dict, table, where=None):
+    cursor = db_connect(db_info_dict).cursor()
+    if where:
+        query = f"""
+            DELETE FROM {table}
+            where {where}
+        """
+    else:
+        query = f"""
+            DELETE FROM {table}
+        """
+    cursor.execute(query)
+    cursor.close()
+
+
+def update_db(db_info_dict, table, set_, where):
+    cursor = db_connect(db_info_dict).cursor()
+    query = f"""
+        update {table}
+        set {set_}
+        where {where}
+    """
+    cursor.execute(query)
+    cursor.close()
+
+
+
+def pd2db(db_host, db_port, db_user, db_passward, 
+          db_name, charset, if_exists, autocommit,
+          df, table, encoding='utf-8-sig', index=False):
+    
+    url = f"mysql+pymysql://{db_user}:{db_passward}@{db_host}:{db_port}/{db_name}?charset={charset}"
+    # engine = create_engine(url, encoding=encoding)
+    engine = create_engine(url)
+    conn = engine.connect()
+    df.to_sql(name=table, con=engine, if_exists=if_exists, index=index)
+    conn.close()
+    
+
+# def pd2db(db_info_dict, df, table, encoding='utf-8-sig', index=False):
+#     db_host = db_info_dict['host']
+#     db_port = db_info_dict['port']
+#     db_user = db_info_dict['user']
+#     db_passward = db_info_dict['passward']
+#     db_name = db_info_dict['name']
+#     charset = db_info_dict['charset']
+#     if_exists = db_info_dict['if_exists']
+#     autocommit = db_info_dict['autocommit']
+#     url = f"mysql+pymysql://{db_user}:{db_passward}@{db_host}:{db_port}/{db_name}?charset={charset}"
+#     # engine = create_engine(url, encoding=encoding)
+#     engine = create_engine(url)
+#     conn = engine.connect()
+#     df.to_sql(name=table, con=engine, if_exists=if_exists, index=index)
+#     conn.close()
+    
+
+"""
+def main():
+    db_host = '211.195.9.226'
+    db_port = 3306
+    db_user = 'root'
+    db_passward = 'theimc#10!'
+    db_name = 'flagship'
+    charset = 'utf8mb4'
+    table = 'test'
+    if_exists = 'append'
+    autocommit = True
+    
+    df = pd.read_csv('D:/python/project/temp/example.csv', encoding='utf-8-sig')
+    
+    pd2db(
+        db_user=db_user, 
+        db_passward=db_passward, 
+        db_host=db_host, 
+        db_port=db_port, 
+        db_name=db_name, 
+        charset=charset, 
+        df=df, 
+        table=table, 
+        if_exists=if_exists, 
+        encoding='utf-8-sig', 
+        index=False
+    )
+    
+
+if __name__ == '__main__':
+    main()
+
+"""
+
+
+
+
+
+
+
+
+
+