WBClean-XUM 1.0__tar.gz

wbclean_xum-1.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: WBClean_XUM
+Version: 1.0
+Summary: A python library that will help developers trying to parse different excel file formats. From conversions of file formats to extracting required information from complex excel sheets, this library is the go-to I would like to propose to all of developer community.
+Author: Muthu Krishnan
+License: GNU
+Project-URL: Repository, https://github.com/nmkrishnan-droid/WBClean_XUM.git
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.0
+Requires-Dist: openpyxl>=3.1.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: pywin32>=306; platform_system == "Windows"

wbclean_xum-1.0/pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "WBClean_XUM"
+version = "1.0"
+description = "A python library that will help developers trying to parse different excel file formats. From conversions of file formats to extracting required information from complex excel sheets, this library is the go-to I would like to propose to all of developer community."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "GNU" }
+authors = [
+  { name = "Muthu Krishnan" }
+]
+dependencies = [
+  "pandas>=2.0",
+  "openpyxl>=3.1.0",
+  "requests>=2.31.0",
+  "pywin32>=306; platform_system=='Windows'"
+]
+
+[project.urls]
+Repository = "https://github.com/nmkrishnan-droid/WBClean_XUM.git"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

wbclean_xum-1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

wbclean_xum-1.0/src/WBClean_XUM/WBClean_XUM.py

@@ -0,0 +1,255 @@
+import re
+import pandas as pd
+from openpyxl import load_workbook, Workbook
+from collections import Counter
+import requests
+import win32com.client as win32
+import os
+
+
+class WBClean_XUM:
+    def XUM_TransposeSheet(self, src_path, src_sheet_name=None, dst_sheet_name=None):
+        wb = load_workbook(src_path, data_only=False)  # set True if you want computed values instead of formulas
+        ws = wb[src_sheet_name] if src_sheet_name else wb.worksheets[0]
+
+        # "Used range" bounds as Excel sees it (approx via openpyxl)
+        min_row, max_row = ws.min_row, ws.max_row
+        min_col, max_col = ws.min_column, ws.max_column
+
+        out_wb = Workbook()
+        out_ws = out_wb.active
+        out_ws.title = dst_sheet_name or (src_sheet_name + "_T")
+
+        # Read every cell in the used rectangle (including blanks) into a grid
+        grid = [
+            [ws.cell(row=r, column=c).value for c in range(min_col, max_col + 1)]
+            for r in range(min_row, max_row + 1)
+        ]
+
+        # Transpose the grid and write it out
+        tgrid = list(zip(*grid))  # tuples
+        for r, row_vals in enumerate(tgrid, start=1):
+            for c, v in enumerate(row_vals, start=1):
+                out_ws.cell(row=r, column=c, value=v)
+
+        print(type(out_wb))
+        tableList = []
+        for row in out_ws.iter_rows():
+            rowList = []
+            for cell in row:
+                print(cell.value, end=" ")
+                rowList.append(cell.value)
+            print("\n")
+            tableList.append(rowList)
+
+        print(f"First transposed table: {tableList}")
+        return tableList
+
+    def XUM_LLMFormat(self, prompt_ReqFeildString, prompt_ReqJSONOutputString, prompt_SampleData, groqModel, Key,
+                      APIUrl,
+                      temperature=0, maxTokens=512, contentType="application/json", prompt_FullCustom=False,
+                      prompt_Full=None):
+        prompt = f"""
+        You are a data-mapping assistant.
+
+        The SAMPLE DATA is given in /{{Key:value}} format where Key is the column Name and the value is sample values for that column
+
+        Map the SAMPLE DATA KEYS to the REQUIRED fields below.
+
+        REQUIRED FIELDS:
+        {prompt_ReqFeildString}
+
+        SAMPLE DATA:
+        {prompt_SampleData}
+
+        RULES:
+        - Choose best column if multiple match
+        - Use null if none apply
+        - alt_cols must be a JSON array
+        - RETURN ONLY VALID JSON. NO TEXT.
+
+        EXPECTED JSON FORMAT:
+        {prompt_ReqJSONOutputString}
+
+        Return ONLY a JSON object. Do not use markdown, do not use code fences, do not add commentary.
+
+        """
+
+        if prompt_FullCustom:
+            prompt = prompt_Full
+
+        payload = {
+            "model": groqModel,
+            "messages": [
+                {"role": "user", "content": prompt}
+            ],
+            "temperature": temperature,
+            "max_tokens": maxTokens
+        }
+
+        r = requests.post(
+            APIUrl,
+            headers={
+                "Authorization": f"Bearer {Key}",
+                "Content-Type": contentType,
+            },
+            json=payload,
+            timeout=60,
+        )
+
+        if r.status_code != 200:
+            raise RuntimeError(
+                f"GROQ ERROR {r.status_code}: {r.text}"
+            )
+
+        content = r.json()["choices"][0]["message"]["content"]
+        try:
+            data = content
+        except Exception as e:
+            raise RuntimeError(f"LLM did not return valid JSON. Raw content:\n{content}") from e
+
+        return data
+
+    def XUM_DeleteRows(self, Table2dArray, rowIndexList):
+        rows_to_drop = set(rowIndexList)  # 0-based indices
+
+        newTable = [
+            row for idx, row in enumerate(Table2dArray)
+            if idx not in rows_to_drop
+        ]
+        return newTable
+
+    def XUM_DeleteColumns(self, Table2dArray, colIndexList):
+        cols_to_drop = set(colIndexList)
+
+        newTable = []
+        for row in Table2dArray:
+            # keep values whose 1-based index is NOT in cols_to_drop
+            newRow = [val for idx, val in enumerate(row, start=1) if idx not in cols_to_drop]
+            newTable.append(newRow)
+
+        return newTable
+
+    def XUM_TextPresenceRegex(self, x, pattern):
+        if x is None:
+            return False
+        return pattern.search(str(x)) is not None
+
+    def XUM_XLSConversion(self, xlsPath):
+        xls_path = os.path.abspath(xlsPath)
+        if xlsPath is None:
+            root, _ = os.path.splitext(xls_path)
+            xlsx_path = root + ".xlsx"
+        xlsxPath = os.path.abspath(xlsPath)
+
+        excel = win32.Dispatch("Excel.Application")
+        excel.Visible = False
+        excel.DisplayAlerts = False
+
+        wb = excel.Workbooks.Open(xls_path)
+        wb.SaveAs(xlsxPath, FileFormat=51)  # 51 = xlOpenXMLWorkbook (.xlsx)
+        wb.Close(False)
+        excel.Quit()
+
+        return xlsxPath
+
+    def XUM_Clean(self, filePath, pattern, returnDF=True, remove_none=True, getImpFeatures=False,
+                  prompt_ReqFeildString=None, prompt_ReqJSONOutputString=None, Key=None,
+                  sheetName=None,
+                  destinationSheet="WBClean_XUM",
+                  APIUrl="https://api.groq.com/openai/v1/chat/completions", groqModel="llama-3.3-70b-versatile",
+                  contentType="application/json", temperature=0, maxTokens=512):
+        transposedList = self.XUM_TransposeSheet(
+            src_path=filePath,
+            src_sheet_name=sheetName,
+            dst_sheet_name=destinationSheet
+        )
+
+        out = []
+        foundIndex = []
+
+        for i in transposedList:
+            for ind, value in enumerate(i, start=1):
+                print(f"value found: {value}")
+                if value == None:
+                    out.append(ind)
+                elif self.XUM_TextPresenceRegex(value, re.compile(rf'\b(?:{pattern})\b', re.I)):
+                    print(f"value matched: {value}")
+                    foundIndex.append(ind)
+                    break
+
+        print(out)
+        print(foundIndex)
+        captureCols = sorted(x for x in out if x < foundIndex[-1])
+        print(f"Caputred! {captureCols}")  # [1, 2]
+
+        freq = Counter(captureCols)
+        print(f"Frequency of col values{freq}")
+        result = sorted(
+            [(num, cnt) for num, cnt in freq.items() if cnt > 1],
+            key=lambda x: x[1],
+            reverse=True
+        )
+        nums_only = [num for num, cnt in result]
+
+        newTable = self.XUM_DeleteColumns(Table2dArray=transposedList, colIndexList=nums_only)
+
+        # TODO: good job Muthu!!! we made it this far!!
+        print(f"this is the new table after deleting unwanted columns: {newTable}")
+
+        headerColumns = [x for i in newTable for ind, x in enumerate(i) if ind == 0]
+        row1Values = [x for i in newTable for ind, x in enumerate(i) if ind == 1]
+
+        if getImpFeatures:
+            testSample = dict(zip(headerColumns, row1Values))
+            columnMapping = self.XUM_LLMFormat(
+                prompt_ReqFeildString=prompt_ReqFeildString,
+                prompt_ReqJSONOutputString=prompt_ReqJSONOutputString,
+                prompt_SampleData=testSample,
+                Key=Key,
+                APIUrl=APIUrl,
+                groqModel=groqModel,
+                temperature=temperature,
+                maxTokens=maxTokens,
+                contentType=contentType
+            )
+
+            rowsWeDontNeed = []
+            for rowInd, row in enumerate(newTable):
+                if row[0] == None:
+                    rowsWeDontNeed.append(rowInd)
+                elif row[0] in columnMapping:
+                    pass
+                else:
+                    rowsWeDontNeed.append(rowInd)
+
+            print("\n")
+            print(f"rows we don't need: {rowsWeDontNeed}")
+
+            TableWeNeed = self.XUM_DeleteRows(Table2dArray=newTable, rowIndexList=rowsWeDontNeed)
+
+            newTable = TableWeNeed
+
+        else:
+            newTable = newTable
+
+        transposeToNormal = [list(r) for r in zip(*newTable)]
+
+        # remove_None = remove_none
+        # rowsToDelete = []
+
+        if remove_none:
+            # for index,rows in enumerate(back):
+            #     for i,j in rows:
+            #         if i==None and j==None:
+            #             rowsToDelete.append(index)
+            rowsToDelete = [idx for idx, row in enumerate(transposeToNormal) if all(v is None for v in row)]
+            table = self.XUM_DeleteRows(Table2dArray=transposeToNormal, rowIndexList=rowsToDelete)
+        else:
+            table = transposeToNormal
+
+        df = pd.DataFrame(table[1:], columns=table[0])
+
+        return df if returnDF else df.to_excel(f"{destinationSheet}.xlsx")
+
+

wbclean_xum-1.0/src/WBClean_XUM.egg-info/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: WBClean_XUM
+Version: 1.0
+Summary: A python library that will help developers trying to parse different excel file formats. From conversions of file formats to extracting required information from complex excel sheets, this library is the go-to I would like to propose to all of developer community.
+Author: Muthu Krishnan
+License: GNU
+Project-URL: Repository, https://github.com/nmkrishnan-droid/WBClean_XUM.git
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.0
+Requires-Dist: openpyxl>=3.1.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: pywin32>=306; platform_system == "Windows"

wbclean_xum-1.0/src/WBClean_XUM.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+README.md
+pyproject.toml
+src/WBClean_XUM/WBClean_XUM.py
+src/WBClean_XUM/__init__.py
+src/WBClean_XUM.egg-info/PKG-INFO
+src/WBClean_XUM.egg-info/SOURCES.txt
+src/WBClean_XUM.egg-info/dependency_links.txt
+src/WBClean_XUM.egg-info/requires.txt
+src/WBClean_XUM.egg-info/top_level.txt

wbclean_xum-1.0/src/WBClean_XUM.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

wbclean_xum-1.0/src/WBClean_XUM.egg-info/requires.txt

@@ -0,0 +1,6 @@
+pandas>=2.0
+openpyxl>=3.1.0
+requests>=2.31.0
+
+[:platform_system == "Windows"]
+pywin32>=306

wbclean_xum-1.0/src/WBClean_XUM.egg-info/top_level.txt

@@ -0,0 +1 @@
+WBClean_XUM