pyBigKinds 1.2.11__tar.gz

pybigkinds-1.2.11/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Jaeseong Choe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pybigkinds-1.2.11/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: pyBigKinds
+Version: 1.2.11
+Summary: BigKinds Data Analysis Toolkit for python
+Author: Sorrychoe
+License: MIT
+Project-URL: Source Code, https://github.com/sorrychoe/pyBigKinds
+Project-URL: Bug Tracker, https://github.com/sorrychoe/pyBigKinds/issues
+Keywords: Journalism,Preprocessing-data,Text-mining,BigKinds
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: matplotlib>=3.5.3
+Requires-Dist: pandas<3,>=2
+Requires-Dist: wordcloud>=1.8.2.2
+Requires-Dist: scikit-learn<2,>=1.0.0
+Requires-Dist: tomotopy>=0.14.0
+Requires-Dist: openpyxl>=3.1.2
+Requires-Dist: mlxtend<0.24,>=0.22.0
+Provides-Extra: dev
+Requires-Dist: build>=0.7; extra == "dev"
+Requires-Dist: flake8>=3.9; extra == "dev"
+Requires-Dist: isort>=5.7; extra == "dev"
+Requires-Dist: pre-commit>=3; extra == "dev"
+Requires-Dist: pylint>=3.2.0; extra == "dev"
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# pyBigKinds
+
+pyBigKinds는 한국 언론의 빅데이터 저장소인 BigKinds에서 추출한 데이터를 low-Code로 분석할 수 있게 만든 툴입니다.
+
+## Requirements
+
+- Python >= 3.10
+- 한글 폰트: Windows(맑은 고딕)와 macOS(AppleGothic)는 기본 폰트를 사용합니다.
+  Linux에서는 `NanumGothic`(예: `sudo apt-get install fonts-nanum`) 등 한글 폰트를
+  설치해야 워드클라우드와 그래프의 한글이 정상 출력됩니다.
+
+## Installation
+
+- macOS
+
+  ```bash
+  python3 -m pip install pyBigKinds
+  ```
+
+- Windows, linux
+
+  ```bash
+  python -m pip install pyBigKinds
+  ```
+
+## Usage
+
+```python
+import pyBigKinds as bk
+
+bk.press_counter(df)
+```
+
+![](https://github.com/sorrychoe/pyBigKinds/blob/release/docs/example1.png)
+
+```python
+import pyBigKinds as bk
+
+bk.keywords_wordcloud(df, "중앙일보")
+```
+
+![](https://github.com/sorrychoe/pyBigKinds/blob/release/docs/example2.png)
+
+
+## License
+
+[MIT](https://choosealicense.com/licenses/mit/)
+
+## you have some issue?
+
+사용 중 문제 발생 시, 해당 Repo issue에 등록해주세요.

pybigkinds-1.2.11/README.md

@@ -0,0 +1,51 @@
+# pyBigKinds
+
+pyBigKinds는 한국 언론의 빅데이터 저장소인 BigKinds에서 추출한 데이터를 low-Code로 분석할 수 있게 만든 툴입니다.
+
+## Requirements
+
+- Python >= 3.10
+- 한글 폰트: Windows(맑은 고딕)와 macOS(AppleGothic)는 기본 폰트를 사용합니다.
+  Linux에서는 `NanumGothic`(예: `sudo apt-get install fonts-nanum`) 등 한글 폰트를
+  설치해야 워드클라우드와 그래프의 한글이 정상 출력됩니다.
+
+## Installation
+
+- macOS
+
+  ```bash
+  python3 -m pip install pyBigKinds
+  ```
+
+- Windows, linux
+
+  ```bash
+  python -m pip install pyBigKinds
+  ```
+
+## Usage
+
+```python
+import pyBigKinds as bk
+
+bk.press_counter(df)
+```
+
+![](https://github.com/sorrychoe/pyBigKinds/blob/release/docs/example1.png)
+
+```python
+import pyBigKinds as bk
+
+bk.keywords_wordcloud(df, "중앙일보")
+```
+
+![](https://github.com/sorrychoe/pyBigKinds/blob/release/docs/example2.png)
+
+
+## License
+
+[MIT](https://choosealicense.com/licenses/mit/)
+
+## you have some issue?
+
+사용 중 문제 발생 시, 해당 Repo issue에 등록해주세요.

pybigkinds-1.2.11/pyBigKinds/__init__.py

@@ -0,0 +1,38 @@
+"""
+BigKindsParser: Exploratory data analysis Toolkit of Python for BigKinds Data
+"""
+
+from . import _version, base, preprocessing, representation, visualization
+from ._version import __version__
+from .base import *
+from .preprocessing import *
+from .representation import *
+from .visualization import *
+
+__all__ = [
+    "press_counter",
+    "header_remover",
+    "day_range",
+    "counter_to_dataframe",
+    "duplication_remover",
+    "keyword_parser",
+    "keyword_list",
+    "word_counter",
+    "keywords_wordcloud",
+    "top_words",
+    "scatterplot",
+    "keyword_dataframe",
+    "keyword_dataframe_no_duplicated",
+    "tfidf",
+    "tfidf_vector",
+    "normalize_vector",
+    "pca",
+    "nmf",
+    "t_sne",
+    "lsa",
+    "kmeans",
+    "dbscan",
+    "meanshift",
+    "lda",
+    "association",
+]

pybigkinds-1.2.11/pyBigKinds/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.11"

pybigkinds-1.2.11/pyBigKinds/base.py

@@ -0,0 +1,152 @@
+# pylint: disable=C0301
+
+import pandas as pd
+
+
+def header_remover(df):
+    """
+    Removes any text enclosed in square brackets ([]) from the '제목' (title) column of a DataFrame or list.
+
+    Parameters:
+    df (pandas.DataFrame or list): The input DataFrame or list containing a column or text data where headers (enclosed in square brackets) need to be removed.
+
+    Returns:
+    pandas.Series or list: A pandas Series or list with the cleaned text where square bracketed sections are removed.
+
+    Raises:
+    TypeError: If the input is not a pandas DataFrame or list.
+    """
+    if isinstance(df, pd.DataFrame):
+        ans = df["제목"].str.replace(r"\[[^\]]*\]", "", regex=True).str.strip()
+    elif isinstance(df, list):
+        ans = pd.Series(df).str.replace(r"\[[^\]]*\]", "", regex=True).str.strip().tolist()
+    else:
+        raise TypeError("input value is to be have to list or DataFrame")
+    return ans
+
+
+def keyword_list(df):
+    """
+    Converts the '키워드' column of a DataFrame to a list or returns a list as-is if the input is already a list or Series.
+
+    Parameters:
+    df (pandas.DataFrame or pandas.Series or list): The input data containing keywords.
+
+    Returns:
+    list: A list of keywords.
+
+    Raises:
+    TypeError: If the input is not a pandas DataFrame, Series, or list.
+    """
+    if isinstance(df, pd.DataFrame):
+        return df["키워드"].values.tolist()
+    elif isinstance(df, pd.Series):
+        return df.values.tolist()
+    elif isinstance(df, list):
+        return df
+    else:
+        raise TypeError("input value is to be have to list, Series or DataFrame")
+
+
+def keyword_parser(text_list):
+    """
+    Parses and splits a list of keywords into individual words by separating them based on commas.
+
+    Parameters:
+    text_list (list): A list of strings where each string contains keywords separated by commas.
+
+    Returns:
+    list of lists: A list where each element is a sublist of keywords split from the original list of strings.
+
+    Raises:
+    TypeError: If the input is not a list.
+    ValueError: If any element in the list is not a valid string format.
+    """
+    if isinstance(text_list, list):
+        news_key = []
+        for word in text_list:
+            if isinstance(word, str):
+                word = word.split(",")
+                news_key.append(word)
+            else:
+                raise ValueError("input list is not valid format")
+        return news_key
+    else:
+        raise TypeError("input type is to be have to list")
+
+
+def duplication_remover(news_key):
+    """
+    Removes duplicate keywords from a list of lists.
+
+    Parameters:
+    news_key (list): A list where each element is a list of keywords.
+
+    Returns:
+    list of lists: A list where duplicate keywords within each sublist have been removed.
+
+    Raises:
+    TypeError: If the input is not a list of lists.
+    ValueError: If any element in the list is not a valid sublist.
+    """
+    if isinstance(news_key, list):
+        news_value = []
+        for j in news_key:
+            if isinstance(j, list):
+                j = list(dict.fromkeys(j))
+                news_value.append(j)
+            else:
+                raise ValueError("input list is not valid format")
+        return news_value
+    else:
+        raise TypeError("input type is to be have to list")
+
+
+def word_counter(news_value):
+    """
+    Counts the occurrence of each keyword from a list of lists of keywords.
+
+    Parameters:
+    news_value (list): A list where each element is a list of keywords.
+
+    Returns:
+    dict: A dictionary where each key is a keyword and each value is the frequency of that keyword.
+
+    Raises:
+    TypeError: If the input is not a list of lists.
+    """
+    if isinstance(news_value, list):
+        key_words = {}
+        for k in range(len(news_value)):
+            for i in news_value[k]:
+                if i not in key_words:
+                    key_words[i] = 1
+                elif i in key_words:
+                    key_words[i] += 1
+        return key_words
+    else:
+        raise TypeError("input type is to be have to list")
+
+
+def counter_to_dataframe(key_words):
+    """
+    Converts a dictionary of keyword counts into a sorted pandas DataFrame.
+
+    Parameters:
+    key_words (dict): A dictionary where each key is a keyword and each value is the frequency of that keyword.
+
+    Returns:
+    pandas.DataFrame: A DataFrame with two columns: '단어' (keyword) and '빈도' (frequency), sorted by frequency in descending order.
+
+    Raises:
+    TypeError: If the input is not a dictionary.
+    """
+    if isinstance(key_words, dict):
+        word_df = pd.DataFrame(key_words.items())
+        word_df.columns = ["단어", "빈도"]
+        word_df = word_df.sort_values(["빈도"], ascending=False).reset_index(
+            drop=True,
+        )
+        return word_df
+    else:
+        raise TypeError("input type is to be have to dict")

pybigkinds-1.2.11/pyBigKinds/preprocessing.py

@@ -0,0 +1,159 @@
+# pylint: disable=C0301
+
+import numpy as np
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import Normalizer
+
+from .base import (
+    counter_to_dataframe,
+    duplication_remover,
+    keyword_list,
+    keyword_parser,
+    word_counter,
+)
+
+
+def keyword_dataframe(df):
+    """
+    Generates a DataFrame of keyword frequencies based on the '키워드' column in the input DataFrame.
+
+    This function extracts keywords from the '키워드' column, parses them, and counts the occurrence of each keyword.
+    The result is returned as a DataFrame with columns for the keyword and its frequency.
+
+    Parameters:
+    df (pandas.DataFrame): The input DataFrame containing a '키워드' column with keywords.
+
+    Returns:
+    pandas.DataFrame: A DataFrame with two columns - '단어' (keyword) and '빈도' (frequency), sorted by frequency in descending order.
+
+    Raises:
+    TypeError: If the input is not a pandas DataFrame.
+    """
+    if isinstance(df, pd.DataFrame):
+        lis = keyword_list(df)
+        keywords = keyword_parser(lis)
+        counter = word_counter(keywords)
+        df = counter_to_dataframe(counter)
+        return df
+    else:
+        raise TypeError("input type is to be have to DataFrame")
+
+
+def keyword_dataframe_no_duplicated(df):
+    """
+    Generates a DataFrame of keyword frequencies after removing duplicate keywords.
+
+    This function extracts keywords from the '키워드' column, removes duplicate keywords within each list, and counts the occurrence of each unique keyword.
+    The result is returned as a DataFrame with columns for the keyword and its frequency.
+
+    Parameters:
+    df (pandas.DataFrame): The input DataFrame containing a '키워드' column with keywords.
+
+    Returns:
+    pandas.DataFrame: A DataFrame with two columns - '단어' (keyword) and '빈도' (frequency), sorted by frequency in descending order.
+
+    Raises:
+    TypeError: If the input is not a pandas DataFrame.
+    """
+    if isinstance(df, pd.DataFrame):
+        lis = keyword_list(df)
+        keywords = keyword_parser(lis)
+        keywords_set = duplication_remover(keywords)
+        counter = word_counter(keywords_set)
+        df = counter_to_dataframe(counter)
+        return df
+    else:
+        raise TypeError("input type is to be have to DataFrame")
+
+
+def tfidf(df, col=None):
+    """
+    Calculates the Term Frequency-Inverse Document Frequency (TF-IDF) for keywords in the input DataFrame.
+
+    This function takes an optional column name (col) to select a specific column for TF-IDF calculations. It uses the TfidfVectorizer to compute TF-IDF values for the keywords
+    and returns a DataFrame of words with their corresponding TF-IDF scores.
+
+    Parameters:
+    df (pandas.DataFrame): The input DataFrame containing text data, typically in a '키워드' column.
+    col (str, optional): A column name specifying which column to apply the TF-IDF transformation. Defaults to None.
+
+    Returns:
+    pandas.DataFrame: A DataFrame with two columns - '단어' (keyword) and '빈도' (TF-IDF score), sorted by score in descending order.
+
+    Raises:
+    TypeError: If the input is not a pandas DataFrame.
+    """
+    if isinstance(df, pd.DataFrame):
+        if isinstance(col, str):
+            df = df[col]
+        lis = keyword_list(df)
+
+        tfidfv = TfidfVectorizer()
+        tdm = tfidfv.fit_transform(lis)
+
+        word_count = (
+            pd.DataFrame(
+                {
+                    "단어": tfidfv.get_feature_names_out(),
+                    "빈도": tdm.sum(axis=0).flat,
+                },
+            )
+            .sort_values("빈도", ascending=False)
+            .reset_index(drop=True)
+        )
+        return word_count
+    else:
+        raise TypeError("input type is to be have to DataFrame")
+
+
+def tfidf_vector(df):
+    """
+    Creates a TF-IDF vector representation of the keywords in the input DataFrame.
+
+    This function uses a scikit-learn Pipeline that combines CountVectorizer and TfidfTransformer to generate a TF-IDF matrix (vectorized form) for the keywords in the DataFrame.
+
+    Parameters:
+    df (pandas.DataFrame): The input DataFrame containing a '키워드' column with keywords.
+
+    Returns:
+    numpy.ndarray: A 2D array where each row corresponds to the TF-IDF vector for the keywords in each row of the DataFrame.
+
+    Raises:
+    TypeError: If the input is not a pandas DataFrame.
+    """
+    if isinstance(df, pd.DataFrame):
+        lis = keyword_list(df)
+        pipeline = Pipeline(
+            [
+                ("vect", CountVectorizer()),
+                ("tfidf", TfidfTransformer()),
+            ],
+        )
+        vec = pipeline.fit_transform(lis).toarray()
+        return vec
+    else:
+        raise TypeError("input type is to be have to DataFrame")
+
+
+def normalize_vector(vec):
+    """
+    Normalizes a TF-IDF vector or any other vector to unit length (L2 norm).
+
+    This function uses the Normalizer from scikit-learn to scale each vector such that the Euclidean norm of each vector becomes 1.
+
+    Parameters:
+    vec (numpy.ndarray): A 2D array where each row is a vector to be normalized.
+
+    Returns:
+    numpy.ndarray: A 2D array with normalized vectors.
+
+    Raises:
+    TypeError: If the input is not a numpy ndarray.
+    """
+    if isinstance(vec, np.ndarray):
+        vec_nor = Normalizer().fit_transform(vec)
+        return vec_nor
+    else:
+        raise TypeError("input type is to be have to ndarray")