py2ls 0.2.5.12__py3-none-any.whl → 0.2.5.14__py3-none-any.whl

py2ls/ips.py

@@ -12,8 +12,7 @@ import re
 import stat
 import platform
 
-from typing import Dict, List, Optional, Union, Any,Tuple
-
+from typing import Dict, List, Optional, Union, Any, Tuple, Literal
 from regex import X
 
 try:
@@ -1663,75 +1662,464 @@ def flatten(nested: Any, unique_list=True, verbose=False):
         return flattened_list
 
 
-# def strcmp(
-#     search_term,
-#     candidates,
-#     ignore_case=True,
-#     get_rank=False,
-#     verbose=False,
-#     scorer="WR",
-#     method=None,
-# ):
-#     """
-#     Compares a search term with a list of candidate strings and finds the best match based on similarity score.
+#! ===========extract_text===========
+def extract_text(
+    text: Union[str, List[str]],
+    patterns: Union[str, List[str]],
+    *,
+    mode: Literal["between", "split", "extract"] = "between",
+    keep: Literal["none", "left", "right", "both", "markers"] = "none",
+    case: Literal["sensitive", "insensitive"] = "insensitive",
+    all_matches: bool = False,
+    positions: bool = False,
+    regex: bool = False,
+    delimiter: Optional[str] = None,
+    trim: bool = True,
+    as_dict: bool = False,
+    verbose: bool = False,
+    **kwargs,
+) -> Union[List[str], Tuple[int, str], Dict[str, Any], List[Dict[str, Any]], None]:
+    """
+    Ultimate text extraction tool with enhanced reliability and features.
 
-#     Parameters:
-#     search_term (str): The term to be searched for.
-#     candidates (list of str): A list of candidate strings to compare against the search term.
-#     ignore_case (bool): If True, the comparison ignores case differences.
-#     verbose (bool): If True, prints the similarity score and the best match.
+    Key improvements:
+    - Robust split mode with proper delimiter handling
+    - Consistent return types across all modes
+    - Improved pattern matching logic
+    - Better edge case handling
 
-#     Returns:
-#     tuple: A tuple containing the best match and its index in the candidates list.
-#     """
-#     from fuzzywuzzy import fuzz, process
-
-#     def to_lower(s, ignore_case=True):
-#         # Converts a string or list of strings to lowercase if ignore_case is True.
-#         if ignore_case:
-#             if isinstance(s, str):
-#                 return s.lower()
-#             elif isinstance(s, list):
-#                 s = [str(i) for i in s]  # convert all to str
-#                 return [elem.lower() for elem in s]
-#         return s
-#     scorer = str(method).lower() if method is not None else scorer
-#     str1_, str2_ = to_lower(search_term, ignore_case), to_lower(candidates, ignore_case)
-#     if isinstance(str2_, list):
-#         if "part" in scorer.lower():
-#             similarity_scores = [fuzz.partial_ratio(str1_, word) for word in str2_]
-#         elif "w" in scorer.lower():
-#             similarity_scores = [fuzz.WRatio(str1_, word) for word in str2_]
-#         elif "ratio" in scorer.lower() or "stri" in scorer.lower():  # Ratio (Strictest)
-#             similarity_scores = [fuzz.ratio(str1_, word) for word in str2_]
-#         else:
-#             similarity_scores = [fuzz.WRatio(str1_, word) for word in str2_]
-#         if get_rank:
-#             idx = [
-#                 similarity_scores.index(i)
-#                 for i in sorted(similarity_scores, reverse=True)
-#             ]
-#             if verbose:
-#                 display([candidates[ii] for ii in idx])
-#             return [candidates[ii] for ii in idx]
-#         best_match_index = similarity_scores.index(max(similarity_scores))
-#         best_match_score = similarity_scores[best_match_index]
-#     else:
-#         best_match_index = 0
-#         if "part" in scorer.lower():
-#             best_match_score = fuzz.partial_ratio(str1_, str2_)
-#         elif "w" in scorer.lower():
-#             best_match_score = fuzz.WRatio(str1_, str2_)
-#         elif "Ratio" in scorer.lower():
-#             best_match_score = fuzz.ratio(str1_, str2_)
-#         else:
-#             best_match_score = fuzz.WRatio(str1_, str2_)
-#     if verbose:
-#         print(f"\nbest_match is: {candidates[best_match_index],best_match_score}")
-#         best_match = process.extract(search_term, candidates)
-#         print(f"建议: {best_match}")
-#     return candidates[best_match_index], best_match_index
 
+    print(extract_text("A,B,C", ",", mode="split", keep="none", all_matches=True))
+    # Correctly returns: ['A', 'B', 'C']
+
+    print(extract_text("A,B,C", ",", mode="split", keep="left"))
+    # Returns: ['A,', 'B,', 'C']
+
+    print(extract_text("A,B,C", ",", mode="split", keep="right"))
+    # Returns: [',B', ',C']
+
+    print(extract_text("A,B,C", ",", mode="split", keep="both"))
+    # Returns: ['A', ',', 'B', ',', 'C']
+    """
+    if verbose:
+        print("""
+extract_text(
+    text: Union[str, List[str]],
+    patterns: Union[str, List[str]],
+    *,
+    mode: Literal["between", "split", "extract"] = "between",
+    keep: Literal["none", "left", "right", "both", "markers"] = "none",
+    case: Literal["sensitive", "insensitive"] = "insensitive",
+    all_matches: bool = False,
+    positions: bool = False,
+    regex: bool = False,
+    delimiter: Optional[str] = None,
+    trim: bool = True,
+    as_dict: bool = False,
+    verbose: bool = False,
+    **kwargs,
+) 
+              """)
+    # Normalization and validation
+    text = _normalize_text(text, delimiter)
+    patterns = _validate_patterns(patterns)
+    flags = re.IGNORECASE if case == "insensitive" else 0
+
+    # Find all matches with enhanced validation
+    matches = _find_matches(text, patterns, regex, flags)
+    if not matches:
+        return None
+
+    # Mode-specific processing
+    if mode == "extract":
+        return _handle_extract(matches, all_matches, as_dict, positions, trim)
+    elif mode == "split":
+        return _handle_split(text, matches, keep, all_matches, as_dict, positions, trim)
+    elif mode == "between":
+        return _handle_between(text, matches, patterns, keep, as_dict, positions, trim)
+    else:
+        raise ValueError(f"Invalid mode: {mode}")
+
+
+def _normalize_text(text: Union[str, List[str]], delimiter: Optional[str]) -> str:
+    """Normalize text input to single string"""
+    if isinstance(text, list):
+        return delimiter.join(text) if delimiter else " ".join(text)
+    return text
+
+
+def _validate_patterns(patterns: Union[str, List[str]]) -> List[str]:
+    """Validate and normalize patterns"""
+    if isinstance(patterns, str):
+        return [patterns]
+    if not patterns:
+        raise ValueError("At least one pattern required")
+    return patterns
+
+
+def _find_matches(
+    text: str, patterns: List[str], regex: bool, flags: int
+) -> List[dict]:
+    """Find all pattern matches with enhanced regex handling"""
+    matches = []
+    for pattern in patterns:
+        try:
+            search_pattern = pattern if regex else re.escape(pattern)
+            for match in re.finditer(search_pattern, text, flags=flags):
+                matches.append(
+                    {
+                        "text": match.group(),
+                        "start": match.start(),
+                        "end": match.end(),
+                        "pattern": pattern,
+                        "full_match": match,
+                    }
+                )
+        except re.error as e:
+            raise ValueError(f"Invalid pattern '{pattern}': {e}")
+    return sorted(matches, key=lambda x: x["start"])
+
+
+def _handle_extract(
+    matches: List[dict], all_matches: bool, as_dict: bool, positions: bool, trim: bool
+) -> Union[List, dict]:
+    """Handle text extraction of matched patterns"""
+    results = []
+    for match in matches if all_matches else [matches[0]]:
+        content = match["text"].strip() if trim else match["text"]
+        result = (
+            {
+                "text": content,
+                "start": match["start"],
+                "end": match["end"],
+                "pattern": match["pattern"],
+            }
+            if as_dict
+            else content
+        )
+        if positions and as_dict:
+            result["positions"] = [(match["start"], match["end"])]
+        results.append(result)
+
+    return results[0] if not all_matches else results
+
+
+def _create_part(
+    content: str,
+    start: int,
+    end: int,
+    match: Optional[dict],
+    as_dict: bool,
+    positions: bool,
+    trim: bool,
+) -> Union[str, dict]:
+    """Create a standardized result part"""
+    content = content.strip() if trim else content
+    if not as_dict:
+        return content
+
+    part = {
+        "text": content,
+        "start": start,
+        "end": end,
+        "pattern": match["pattern"] if match else None,
+    }
+    if positions and match:
+        part["positions"] = [(match["start"], match["end"])]
+    return part
+
+
+def _handle_between(
+    text: str,
+    matches: List[dict],
+    patterns: List[str],
+    keep: str,
+    as_dict: bool,
+    positions: bool,
+    trim: bool,
+) -> Union[Tuple, dict]:
+    """Reliable between-mode implementation with boundary checks"""
+    first_pattern, last_pattern = patterns[0], patterns[-1]
+    first_matches = [m for m in matches if m["pattern"] == first_pattern]
+    last_matches = [m for m in matches if m["pattern"] == last_pattern]
+
+    if not first_matches or not last_matches:
+        return None
+
+    first = first_matches[0]
+    last = last_matches[-1]
+
+    if first["start"] > last["start"]:
+        return None
+
+    # Calculate extraction window
+    start, end = first["start"], last["end"]
+    if keep == "none":
+        start, end = first["end"], last["start"]
+    elif keep == "left":
+        end = last["start"]
+    elif keep == "right":
+        start = first["end"]
+
+    extracted = text[start:end].strip() if trim else text[start:end]
+
+    if as_dict:
+        result = {
+            "text": extracted,
+            "start": start,
+            "end": end,
+            "patterns": patterns,
+            "match_positions": [(m["start"], m["end"]) for m in matches],
+        }
+        return result
+
+    return (
+        (start, extracted)
+        if not positions
+        else (start, extracted, [(m["start"], m["end"]) for m in matches])
+    )
+
+
+def _handle_split(
+    text: str,
+    matches: List[dict],
+    keep: str,
+    all_matches: bool,
+    as_dict: bool,
+    positions: bool,
+    trim: bool,
+) -> Union[List, dict]:
+    """Split text with proper handling of keep='both' to include delimiters on both sides"""
+    if not matches:
+        return (
+            [text]
+            if not as_dict
+            else [{"text": text, "start": 0, "end": len(text), "pattern": None}]
+        )
+
+    parts = []
+    prev_end = 0
+    process_matches = matches if all_matches else [matches[0]]
+
+    # Special handling for keep="both"
+    if keep == "both":
+        for i, match in enumerate(process_matches):
+            start, end = match["start"], match["end"]
+            matched_text = text[start:end]
+
+            # First segment (text before first delimiter + first delimiter)
+            if i == 0:
+                segment = text[prev_end:end]  # From start to end of first delimiter
+                if trim:
+                    segment = segment.strip()
+                if segment or not trim:
+                    if as_dict:
+                        parts.append(
+                            {
+                                "text": segment,
+                                "start": prev_end,
+                                "end": end,
+                                "pattern": match["pattern"],
+                                **({"positions": [(start, end)]} if positions else {}),
+                            }
+                        )
+                    else:
+                        parts.append(segment)
+                prev_end = end
+
+            # Middle segments (delimiter + text + next delimiter)
+            if i > 0 and i < len(process_matches):
+                next_match = process_matches[i]
+                next_start, next_end = next_match["start"], next_match["end"]
+                segment = text[
+                    prev_end:next_end
+                ]  # From prev_end to end of next delimiter
+                if trim:
+                    segment = segment.strip()
+                if segment or not trim:
+                    if as_dict:
+                        parts.append(
+                            {
+                                "text": segment,
+                                "start": prev_end,
+                                "end": next_end,
+                                "pattern": next_match["pattern"],
+                                **(
+                                    {"positions": [(next_start, next_end)]}
+                                    if positions
+                                    else {}
+                                ),
+                            }
+                        )
+                    else:
+                        parts.append(segment)
+                prev_end = next_end
+
+        # Last segment (last delimiter + remaining text)
+        if process_matches and prev_end < len(text):
+            last_match = process_matches[-1]
+            segment = text[
+                last_match["start"] : len(text)
+            ]  # From last delimiter to end
+            if trim:
+                segment = segment.strip()
+            if segment or not trim:
+                if as_dict:
+                    parts.append(
+                        {
+                            "text": segment,
+                            "start": last_match["start"],
+                            "end": len(text),
+                            "pattern": last_match["pattern"],
+                            **(
+                                {
+                                    "positions": [
+                                        (last_match["start"], last_match["end"])
+                                    ]
+                                }
+                                if positions
+                                else {}
+                            ),
+                        }
+                    )
+                else:
+                    parts.append(segment)
+
+        return parts
+
+    # Original handling for other keep modes
+    for i, match in enumerate(process_matches):
+        start, end = match["start"], match["end"]
+        matched_text = text[start:end]
+
+        # Handle text before the match
+        if prev_end < start:
+            before = text[prev_end:start]
+            if trim:
+                before = before.strip()
+            if before or not trim:
+                if as_dict:
+                    parts.append(
+                        {
+                            "text": before,
+                            "start": prev_end,
+                            "end": start,
+                            "pattern": None,
+                            **({"positions": []} if positions else {}),
+                        }
+                    )
+                else:
+                    parts.append(before)
+
+        # Handle the match based on keep mode
+        if keep == "none":
+            pass  # Skip the delimiter
+        elif keep == "left":
+            if parts:
+                if as_dict:
+                    parts[-1]["text"] += matched_text
+                    parts[-1]["end"] = end
+                else:
+                    parts[-1] += matched_text
+            else:
+                if as_dict:
+                    parts.append(
+                        {
+                            "text": matched_text,
+                            "start": start,
+                            "end": end,
+                            "pattern": match["pattern"],
+                            **({"positions": [(start, end)]} if positions else {}),
+                        }
+                    )
+                else:
+                    parts.append(matched_text)
+        elif keep == "right":
+            if i < len(process_matches) - 1:
+                next_start = process_matches[i + 1]["start"]
+                if end < next_start:
+                    between = text[end:next_start]
+                    if as_dict:
+                        parts.append(
+                            {
+                                "text": matched_text + between,
+                                "start": start,
+                                "end": next_start,
+                                "pattern": match["pattern"],
+                                **({"positions": [(start, end)]} if positions else {}),
+                            }
+                        )
+                    else:
+                        parts.append(matched_text + between)
+                    prev_end = next_start
+                    continue
+
+        prev_end = end
+
+    # Handle remaining text after last match
+    if prev_end < len(text):
+        remaining = text[prev_end:]
+        if trim:
+            remaining = remaining.strip()
+        if remaining or not trim:
+            if keep == "right" and parts and process_matches:
+                last_match = process_matches[-1]
+                matched_text = text[last_match["start"] : last_match["end"]]
+                if as_dict:
+                    parts.append(
+                        {
+                            "text": matched_text + remaining,
+                            "start": last_match["start"],
+                            "end": len(text),
+                            "pattern": last_match["pattern"],
+                            **(
+                                {
+                                    "positions": [
+                                        (last_match["start"], last_match["end"])
+                                    ]
+                                }
+                                if positions
+                                else {}
+                            ),
+                        }
+                    )
+                else:
+                    parts.append(matched_text + remaining)
+            else:
+                if as_dict:
+                    parts.append(
+                        {
+                            "text": remaining,
+                            "start": prev_end,
+                            "end": len(text),
+                            "pattern": None,
+                            **({"positions": []} if positions else {}),
+                        }
+                    )
+                else:
+                    parts.append(remaining)
+
+    # Filter empty parts if trimming
+    if trim:
+        parts = [p for p in parts if (p["text"].strip() if as_dict else p.strip())]
+
+    return parts
+
+
+def _merge_parts(
+    parts: List[Union[str, dict]], text: str, as_dict: bool, trim: bool
+) -> Union[str, dict]:
+    """Merge adjacent parts for keep=left mode"""
+    if as_dict:
+        merged_text = "".join(p["text"] for p in parts)
+        return {
+            "text": merged_text.strip() if trim else merged_text,
+            "start": parts[0]["start"],
+            "end": parts[-1]["end"],
+            "patterns": list(set(p["pattern"] for p in parts if p["pattern"])),
+        }
+    return "".join(parts).strip() if trim else "".join(parts)
+#! ===========extract_text===========
 
 def strcmp(
     search_term: str,
@@ -2794,73 +3182,6 @@ def text2audio(
 
 # from datetime import datetime
 from dateutil import parser
-# import re
-# from typing import Union, Optional, Dict, Any
-# def str2time(time_str, fmt="24"):
-#     """
-#     Convert a time string into the specified format.
-#     Parameters:
-#     - time_str (str): The time string to be converted.
-#     - fmt (str): The format to convert the time to. Defaults to '%H:%M:%S'.
-#     Returns:
-#         %I represents the hour in 12-hour format.
-#         %H represents the hour in 24-hour format (00 through 23).
-#         %M represents the minute.
-#         %S represents the second.
-#         %p represents AM or PM.
-#     - str: The converted time string.
-#     """
-#     from datetime import datetime
-
-#     def time_len_corr(time_str):
-#         time_str_ = (
-#             ssplit(time_str, by=[":", " ", "digital_num"]) if ":" in time_str else None
-#         )
-#         time_str_split = []
-#         [time_str_split.append(i) for i in time_str_ if is_num(i)]
-#         if time_str_split:
-#             if len(time_str_split) == 2:
-#                 H, M = time_str_split
-#                 time_str_full = H + ":" + M + ":00"
-#             elif len(time_str_split) == 3:
-#                 H, M, S = time_str_split
-#                 time_str_full = H + ":" + M + ":" + S
-#         else:
-#             time_str_full = time_str_
-#         if "am" in time_str.lower():
-#             time_str_full += " AM"
-#         elif "pm" in time_str.lower():
-#             time_str_full += " PM"
-#         return time_str_full
-
-#     if "12" in fmt:
-#         fmt = "%I:%M:%S %p"
-#     elif "24" in fmt:
-#         fmt = "%H:%M:%S"
-
-#     try:
-#         # Try to parse the time string assuming it could be in 24-hour or 12-hour format
-#         time_obj = datetime.strptime(time_len_corr(time_str), "%H:%M:%S")
-#     except ValueError:
-#         try:
-#             time_obj = datetime.strptime(time_len_corr(time_str), "%I:%M:%S %p")
-#         except ValueError as e:
-#             raise ValueError(f"Unable to parse time string: {time_str}. Error: {e}")
-
-#     # Format the time object to the desired output format
-#     formatted_time = time_obj.strftime(fmt)
-#     return formatted_time
-
-
-# # # Example usage:
-# # time_str1 = "14:30:45"
-# # time_str2 = "02:30:45 PM"
-
-# # formatted_time1 = str2time(time_str1, fmt='12')  # Convert to 12-hour format
-# # formatted_time2 = str2time(time_str2, fmt='24')    # Convert to 24-hour format
-
-# # print(formatted_time1)  # Output: 02:30:45 PM
-# # print(formatted_time2)  # Output: 14:30:45
 def str2time(
     time_str: str, 
     fmt: str = "24", 
@@ -2964,57 +3285,6 @@ def str2time(
             raise ValueError(f"Unable to parse time string: '{time_str}'. Error: {e}")
         return default
 
-
-# def str2date(date_str, original_fmt=None, fmt="%Y-%m-%d"):
-#     """
-#     Convert a date string to the desired format and extract components if needed.
-#     Usage:
-#         str2date(x, fmt="%d.%m.%y",original_fmt="%d.%m.%y")
-#     Parameters:
-#     - date_str (str): The input date string.
-#     - original_fmt (str, optional): The original format of the date string. If not provided, it will be auto-detected.
-#     - fmt (str): The desired format for the output date string. Defaults to '%Y-%m-%d'.
-
-#     Returns:
-#     - dict: A dictionary containing the converted date string and its components (year, month, day).
-    
-#     Raises:
-#     - ValueError: If the date cannot be parsed.
-#     """ 
-#     from dateutil import parser
-#     try:
-#         if not isinstance(date_str,str):
-#             date_str=str(date_str) 
-#         # Parse the date using the provided original format or auto-detect
-#         if original_fmt:
-#             try:
-#                 date_obj = datetime.strptime(date_str, original_fmt)
-#             except Exception as e:
-#                 print(e)
-#                 date_obj=None
-#         else:
-#             try:
-#                 date_obj = parser.parse(date_str)
-#             except Exception as e:
-#                 print(e)
-#                 date_obj=None
-#         # Return formatted string if `fmt` is specified, otherwise return the datetime object
-#         if date_obj is not None:
-#             if fmt: 
-#                 date_obj=date_obj.strftime(fmt)
-#         else:
-#             date_obj=date_str
-#         return date_obj
-
-#     except (ValueError, TypeError) as e:
-#         raise ValueError(f"Unable to process date string: '{date_str}'. Error: {e}")
-
-
-# # str1=str2date(num2str(20240625),fmt="%a %d-%B-%Y")
-# # print(str1)
-# # str2=str2num(str2date(str1,fmt='%a %Y%m%d'))
-# # print(str2)
- 
 def str2date(
     date_str: Union[str, int, float], 
     fmt: Optional[str] = "%Y-%m-%d",
@@ -4054,8 +4324,7 @@ def pdf2ppt(dir_pdf, dir_ppt):
 
 
 def ssplit(text, by="space", verbose: bool =False, strict: bool =False, strip_results: bool = True, **kws):
-    """
-    #         Determines the splitting strategy:
+    """# Determines the splitting strategy:
     #         - "space", "whitespace", "sp": split by whitespace (default)
     #         - "word": split into words using NLTK's word_tokenize
     #         - "sentence", "sent": split into sentences using NLTK's sent_tokenize
@@ -4172,13 +4441,6 @@ def ssplit(text, by="space", verbose: bool =False, strict: bool =False, strip_re
 
     def split_by_regex_end(text, pattern):
         return re.split(f"(?={pattern})", text)
-
-    # def split_by_sentence_endings(text):
-    #     return re.split(r"(?<=[.!?])", text)
-    # def split_non_ascii(text):
-    #     # return re.split(r"([^\x00-\x7F\w\s,.!?:\"'()\-]+)", text)
-    #     # return re.split(r"[^\x00-\x7F]+", text)
-    #     return re.split(r"([^\x00-\x7F]+)", text)
     def split_non_ascii(text, keep_delimiters=False):
         """
         Split text at non-ASCII characters.
@@ -4903,145 +5165,6 @@ def _backup_validations(sheet, verbose=False):
 
     return backup
 
-# def _backup_validations(sheet):
-#     """
-#     Complete validation backup with XML-level cross-sheet detection
-#     """
-#     from openpyxl.utils import get_column_letter
-#     import re
-#     from openpyxl.worksheet.datavalidation import DataValidation
-#     from openpyxl.xml.functions import fromstring
-
-#     backup = {
-#         "validations": [],
-#         "conditional_formatting": [],
-#         "merged_cells": [str(mr) for mr in sheet.merged_cells.ranges],
-#         "_metadata": {
-#             "validated_cells": set(),
-#             "validated_columns": set(),
-#             "validation_types": set(),
-#             "cross_sheet_validations": set()
-#         }
-#     }
-
-#     # METHOD 1: Primary validation backup (standard method)
-#     for dv in sheet.data_validations:
-#         # ... (existing standard validation backup code) ...
-
-#     # METHOD 2: XML-based cross-sheet validation detection
-#     print("Performing deep XML scan for cross-sheet validations...")
-    
-#     # Access the worksheet XML directly
-#     xml_source = sheet._worksheet.xml
-#     if not xml_source:
-#         print("Warning: Could not access worksheet XML source")
-#         return backup
-
-#     try:
-#         # Parse the XML
-#         root = fromstring(xml_source)
-#         ns = {'ns': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
-        
-#         # Find all dataValidation elements
-#         for dv_xml in root.findall('.//ns:dataValidation', ns):
-#             try:
-#                 # Extract validation attributes
-#                 dv_type = dv_xml.get('type', 'none')
-#                 formula1 = dv_xml.find('.//ns:formula1', ns)
-#                 formula_text = formula1.text if formula1 is not None else None
-                
-#                 # Skip if not a list type or no formula
-#                 if dv_type != 'list' or not formula_text:
-#                     continue
-                
-#                 # Clean the formula
-#                 clean_formula = formula_text.strip('"\'')
-                
-#                 # Check for cross-sheet patterns
-#                 cross_sheet_patterns = [
-#                     (r'^[\w\s]+!\$?[A-Za-z]+\$?\d+(?::\$?[A-Za-z]+\$?\d+)?$', "direct sheet reference"),
-#                     (r'INDIRECT\(["\'][\w\s]+![A-Za-z]+\d+(?::[A-Za-z]+\d+)?["\']\)', "INDIRECT sheet reference"),
-#                     (r'^[^\s!]+$', "potential named range"),
-#                 ]
-                
-#                 # Determine if this is a cross-sheet reference
-#                 is_cross_sheet = False
-#                 detection_method = ""
-                
-#                 for pattern, description in cross_sheet_patterns:
-#                     if re.match(pattern, clean_formula, re.IGNORECASE):
-#                         is_cross_sheet = True
-#                         detection_method = description
-#                         break
-                
-#                 if not is_cross_sheet:
-#                     continue
-                
-#                 # Process the ranges
-#                 ranges = []
-#                 sqref = dv_xml.get('sqref', '')
-#                 for range_str in sqref.split():
-#                     try:
-#                         # Convert range to coordinates
-#                         if ':' in range_str:
-#                             start, end = range_str.split(':')
-#                             col_start = int(''.join(filter(str.isdigit, start)))
-#                             col_end = int(''.join(filter(str.isdigit, end)))
-#                             row_start = int(''.join(filter(str.isalpha, start)))
-#                             row_end = int(''.join(filter(str.isalpha, end)))
-#                             ranges.append({
-#                                 'range': range_str,
-#                                 'cells': [f"{get_column_letter(col)}{row}" 
-#                                          for col in range(col_start, col_end+1)
-#                                          for row in range(row_start, row_end+1)]
-#                             })
-#                         else:
-#                             col = int(''.join(filter(str.isdigit, range_str)))
-#                             row = int(''.join(filter(str.isalpha, range_str)))
-#                             ranges.append({
-#                                 'range': range_str,
-#                                 'cells': [f"{get_column_letter(col)}{row}"]
-#                             })
-#                     except Exception as e:
-#                         print(f"Error parsing range {range_str}: {e}")
-                
-#                 # Create validation record
-#                 validation_data = {
-#                     'type': 'list',
-#                     'formula1': formula_text,
-#                     'formula2': None,
-#                     'allow_blank': dv_xml.get('allowBlank', '1') == '1',
-#                     'showDropDown': dv_xml.get('showDropDown', '1') == '1',
-#                     'showInputMessage': dv_xml.get('showInputMessage', '1') == '1',
-#                     'showErrorMessage': dv_xml.get('showErrorMessage', '0') == '1',
-#                     'errorTitle': dv_xml.get('errorTitle', ''),
-#                     'error': dv_xml.get('error', ''),
-#                     'promptTitle': dv_xml.get('promptTitle', ''),
-#                     'prompt': dv_xml.get('prompt', ''),
-#                     'ranges': ranges,
-#                     '_source': 'xml_validation',
-#                     '_detection_method': detection_method,
-#                     '_is_cross_sheet': True,
-#                     '_formula_clean': clean_formula
-#                 }
-                
-#                 # Add to backup
-#                 backup['validations'].append(validation_data)
-#                 for rng in ranges:
-#                     for cell_ref in rng['cells']:
-#                         backup['_metadata']['validated_cells'].add(cell_ref)
-#                         backup['_metadata']['validated_columns'].add(''.join(filter(str.isalpha, cell_ref)))
-#                 backup['_metadata']['validation_types'].add('list')
-#                 backup['_metadata']['cross_sheet_validations'].add(clean_formula.split('!')[0])
-                
-#             except Exception as e:
-#                 print(f"Error processing XML validation: {e}")
-                
-#     except Exception as e:
-#         print(f"Error parsing worksheet XML: {e}")
-    
-#     return backup
-
 def _restore_validations(sheet, backup,verbose=False):
     """
     恢复数据验证和条件格式规则到工作表
@@ -5247,11 +5370,6 @@ def fload(fpath, kind=None, **kwargs):
         with open(fpath, "r") as file:
             content = file.read()
         return content
-
-    # def load_html(fpath):
-    #     with open(fpath, "r") as file:
-    #         content = file.read()
-    #     return content
     def load_html(fpath, **kwargs):
         return pd.read_html(fpath, **kwargs)
 
@@ -7118,7 +7236,7 @@ def listdir(
     hidden=False,  # Include hidden files/folders
     orient="list",
     output="df",  # "df", 'list','dict','records','index','series'
-    verbose=True,
+    verbose=False,
 ):
     def is_hidden(filepath):
         """Check if a file or folder is hidden."""
@@ -7348,7 +7466,7 @@ def listdir(
         if "se" in orient.lower():  # records
             return Box(f.to_dict(orient="series"))
 
-
+ 
 def listpkg(where="env", verbose=False):
     """list all pacakages"""
 
@@ -7829,87 +7947,7 @@ def split_path(fpath):
     dir_par = f_slash.join(fpath.split(f_slash)[:-1])
     dir_ch = "".join(fpath.split(f_slash)[-1:])
     return dir_par, dir_ch
-
-
-def figsave(*args, dpi=300, **kwargs): 
-    import matplotlib.pyplot as plt
-    from PIL import Image
-    bbox_inches = kwargs.pop("bbox_inches", "tight")
-    pad_inches = kwargs.pop("pad_inches", 0)
-    facecolor = kwargs.pop("facecolor", "white")
-    edgecolor = kwargs.pop("edgecolor", "auto")
-
-    dir_save = None
-    fname = None
-    img = None
-
-    for arg in args:
-        if isinstance(arg, str):
-            path = Path(arg)
-            if path.suffix:  # Has file extension
-                fname = path.name
-                dir_save = path.parent
-            else:
-                dir_save = path
-        elif isinstance(arg, (Image.Image, np.ndarray)):
-            img = arg  # Store PIL image or numpy array
-
-    # Set default save directory
-    dir_save = Path(dir_save) if dir_save else Path(".")
-    dir_save.mkdir(parents=True, exist_ok=True)
-
-    # Handle filename and extension
-    if fname is None:
-        fname = "figure"
-    fname = dir_save / fname
-    if fname.suffix == "":
-        fname = fname.with_suffix(".pdf")  # Default format
-
-    ftype = fname.suffix.lstrip(".").lower()
-
-    # Save figure based on file type
-    if ftype == "eps":
-        plt.savefig(fname, format="eps", bbox_inches=bbox_inches)
-        plt.savefig(fname.with_suffix(".pdf"), format="pdf", dpi=dpi,
-                    pad_inches=pad_inches, bbox_inches=bbox_inches,
-                    facecolor=facecolor, edgecolor=edgecolor)
-    elif ftype == "pdf":
-        plt.savefig(fname, format="pdf", dpi=dpi, pad_inches=pad_inches,
-                    bbox_inches=bbox_inches, facecolor=facecolor, edgecolor=edgecolor)
-    elif ftype in ["jpg", "jpeg", "png", "tiff", "tif"]:
-        if img is not None:  # If an image is provided
-            if isinstance(img, Image.Image):
-                img = img.convert("RGB") if img.mode == "RGBA" else img
-                img.save(fname, format=ftype.upper(), dpi=(dpi, dpi))
-            elif isinstance(img, np.ndarray):
-                import cv2
-                if img.ndim == 2:
-                    Image.fromarray(img).save(fname, format=ftype.upper(), dpi=(dpi, dpi))
-                elif img.ndim == 3:
-                    if img.shape[2] == 3:
-                        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-                    elif img.shape[2] == 4:
-                        img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGBA)
-                    Image.fromarray(img).save(fname, format=ftype.upper(), dpi=(dpi, dpi))
-                else:
-                    raise ValueError("Unexpected image dimensions.")
-        else:
-            plt.savefig(fname, format=ftype, dpi=dpi, pad_inches=pad_inches,
-                        bbox_inches=bbox_inches, facecolor=facecolor, edgecolor=edgecolor)
-    elif ftype == "ico":
-        if img is None:
-            plt.savefig(fname, dpi=dpi, pad_inches=pad_inches,
-                        bbox_inches=bbox_inches, facecolor=facecolor, edgecolor=edgecolor)
-            img = Image.open(fname)
-        img = img.convert("RGBA")
-        icon_sizes = [(32, 32), (64, 64), (128, 128), (256, 256)]
-        img.save(fname, format="ICO", sizes=icon_sizes)
-        print(f"Icon saved @: {fname} with sizes: {icon_sizes}")
-    else:
-        raise ValueError(f"Unsupported file format: {ftype}")
-
-    print(f"\nSaved @ {fname} (dpi={dpi})")
-
+ 
 def figsave(*args, dpi=300, **kwargs):
     """
     Save a Matplotlib figure or image file in various formats.
@@ -8038,7 +8076,7 @@ def figsave(*args, dpi=300, **kwargs):
         img = img.convert("RGBA") 
         img.save(fname, format="ICO", sizes=icon_sizes)
         print(f"Icon saved @: {fname} with sizes: {icon_sizes}")
-    print(f"\n✅ Saved @: dpi={dpi}\n{fname}")
+    print(f"\nSaved @: dpi={dpi}\n{fname}")
 
 
 def is_str_color(s):
@@ -8806,7 +8844,8 @@ def detect_angle(image, by="median", template=None):
 
     # Use Hough transform to detect lines
     lines = transform.probabilistic_hough_line(edges)
-
+    if isinstance(by, bool):
+        by="mean" if by else 0
     if not lines and any(["me" in by, "pca" in by]):
         print("No lines detected. Adjust the edge detection parameters.")
         return 0
@@ -9180,7 +9219,7 @@ def imgsets(
         elif "cro" in k.lower() or "cut" in k.lower():
             img_update = img_update.crop(value)
         elif "rota" in k.lower():
-            if isinstance(value, str):
+            if isinstance(value, (str,bool)):
                 value = detect_angle(img_update, by=value)
                 print(f"rotated by {value}°")
             img_update = img_update.rotate(value)
@@ -9524,11 +9563,252 @@ def finfo(fpath, output='json', verbose=False):
         extra_info=extra_info,
     )
 
-
-
+def color2rgb(
+    color_input: str | tuple | list | None, 
+    alpha: float | None = None
+) -> tuple | None:
+    """
+    Ultimate color conversion utility with support for multiple formats and transparency.
+    
+    Parameters:
+    -----------
+    color_input : str | tuple | list | None
+        Supported formats:
+        - Hex strings ("#RRGGBB", "#RGB")
+        - Named colors ("red", "blue")
+        - RGB tuples ((0.2, 0.4, 0.6))
+        - RGBA tuples ((0.2, 0.4, 0.6, 0.8))
+        - HTML/CSS colors ("cornflowerblue")
+        - CSS formats:
+          - rgb(100,200,50)
+          - rgba(100,200,50,0.8)
+          - hsl(120,60%,70%)
+          - hsla(120,60%,70%,0.8)
+    alpha : float | None, optional
+        Opacity value (0.0-1.0). If provided, adds/overrides alpha channel.
+    
+    Returns:
+    --------
+    tuple | None
+        (R, G, B) or (R, G, B, A) tuple in 0-1 range, or None if invalid
+    """
+    from matplotlib import colors as mcolors
+    import re
+    
+    if color_input is None:
+        return None
+    
+    # Case 1: Already in RGB/RGBA tuple format
+    if isinstance(color_input, (tuple, list)):
+        if 3 <= len(color_input) <= 4:
+            if all(0 <= x <= 1 for x in color_input):
+                if alpha is not None and len(color_input) == 3:
+                    return (*color_input, alpha)
+                return tuple(color_input)
+    
+    # Case 2: String input
+    if isinstance(color_input, str):
+        # Remove whitespace and make lowercase
+        color_str = color_input.strip().lower()
+        
+        # Handle CSS rgb/rgba format
+        if color_str.startswith(('rgb(', 'rgba(')):
+            try:
+                nums = list(map(float, re.findall(r"[\d.]+", color_str)))
+                if 3 <= len(nums) <= 4:
+                    rgb = tuple(x/255 if i < 3 else x for i, x in enumerate(nums))
+                    if alpha is not None:
+                        return (*rgb[:3], alpha)
+                    return rgb[:4] if len(rgb) == 4 else rgb[:3]
+            except:
+                pass
+        
+        # Handle CSS hsl/hsla format
+        elif color_str.startswith(('hsl(', 'hsla(')):
+            try:
+                nums = list(map(float, re.findall(r"[\d.]+", color_str)))
+                if 3 <= len(nums) <= 4:
+                    h, s, l = nums[0]/360, nums[1]/100, nums[2]/100
+                    rgb = mcolors.hsv_to_rgb((h, s, l))
+                    if len(nums) == 4:
+                        rgb += (nums[3],)
+                    if alpha is not None:
+                        return (*rgb[:3], alpha)
+                    return rgb[:4] if len(rgb) == 4 else rgb[:3]
+            except:
+                pass
+        
+        # Standard hex/named color processing
+        try:
+            rgb = mcolors.to_rgba(color_str)
+            if alpha is not None:
+                return (*rgb[:3], alpha)
+            return rgb if len(rgb) == 4 and rgb[3] != 1 else rgb[:3]
+        except ValueError:
+            pass
+    
+    # Fallback for invalid colors
+    print(f"Warning: Invalid color format '{color_input}'")
+    return None
+ 
+def color2hex(
+    color_input: str | tuple | list | dict | int | None,
+    keep_alpha: bool = False,
+    force_long: bool = False,
+    uppercase: bool = False,
+    prefix: str = "#",
+    allow_short: bool = True
+) -> str | None:
+    """
+    Ultimate color to hex converter with comprehensive format support.
+    
+    Parameters:
+    -----------
+    color_input : str | tuple | list | dict | int | None
+        Input color in any of these formats:
+        - Hex strings ("#RRGGBB", "#RGB", "RRGGBB", "RGB")
+        - Named colors ("red", "blue", "transparent")
+        - RGB/RGBA tuples ((0.2, 0.4, 0.6), (255, 0, 0), (100, 100, 100, 0.5))
+        - CSS formats:
+          - rgb(100,200,50)
+          - rgba(100,200,50,0.8)
+          - hsl(120,60%,70%)
+          - hsla(120,60%,70%,0.8)
+        - Integer RGB (0xFF0000 for red)
+        - Dictionary {"r": 255, "g": 0, "b": 0} or {"h": 0, "s": 100, "l": 50}
+    keep_alpha : bool, optional
+        Whether to include alpha channel in hex format (#RRGGBBAA)
+    force_long : bool, optional
+        Force 6/8-digit hex even when 3/4-digit would be possible
+    uppercase : bool, optional
+        Use uppercase hex characters (False for lowercase)
+    prefix : str, optional
+        Prefix for hex string ("#" for CSS, "0x" for programming, "" for raw)
+    allow_short : bool, optional
+        Allow shortened 3/4-digit hex when possible
+    
+    Returns:
+    --------
+    str | None
+        Hex color string or None if invalid
+        
+    Examples:
+    ---------
+    >>> color2hex((0.5, 0.2, 0.8)) → "#7f33cc"
+    >>> color2hex("rgb(127, 51, 204)") → "#7f33cc"
+    >>> color2hex((0.2, 0.4, 0.6, 0.8), True) → "#336699cc"
+    >>> color2hex(0xFF0000, uppercase=True) → "#FF0000"
+    >>> color2hex({"r": 255, "g": 165, "b": 0}, prefix="") → "ffa500"
+    >>> color2hex("hsl(120, 100%, 50%)") → "#00ff00"
+    """
+    from matplotlib import colors as mcolors
+    import re
+    
+    def to_rgba(color) -> tuple | None:
+        """Internal conversion to RGBA tuple"""
+        # Handle None
+        if color is None:
+            return None
+            
+        # Handle integer RGB
+        if isinstance(color, int):
+            if color < 0:
+                return None
+            return (
+                (color >> 16) & 0xFF,
+                (color >> 8) & 0xFF,
+                color & 0xFF,
+                255
+            )
+        
+        # Handle dictionary formats
+        if isinstance(color, dict):
+            keys = set(color.keys())
+            if {'r','g','b'}.issubset(keys):
+                return (
+                    color['r'] / 255 if color['r'] > 1 else color['r'],
+                    color['g'] / 255 if color['g'] > 1 else color['g'],
+                    color['b'] / 255 if color['b'] > 1 else color['b'],
+                    color.get('a', 1.0)
+                )
+            elif {'h','s','l'}.issubset(keys):
+                return mcolors.hsv_to_rgb((
+                    color['h'] / 360,
+                    color['s'] / 100,
+                    color['l'] / 100
+                )) + (color.get('a', 1.0),)
+            return None
+        
+        # Handle string formats
+        if isinstance(color, str):
+            color = color.strip().lower()
+            
+            # Handle hex without prefix
+            if re.match(r'^[0-9a-f]{3,8}$', color):
+                return mcolors.to_rgba(f"#{color}")
+                
+            # Handle CSS functions
+            if color.startswith(('rgb(', 'rgba(', 'hsl(', 'hsla(')):
+                try:
+                    return mcolors.to_rgba(color)
+                except ValueError:
+                    return None
+                    
+            # Handle named colors (including 'transparent')
+            try:
+                return mcolors.to_rgba(color)
+            except ValueError:
+                return None
+        
+        # Handle tuple/list formats
+        if isinstance(color, (tuple, list)):
+            if len(color) in (3, 4):
+                # Normalize values
+                normalized = []
+                for i, v in enumerate(color):
+                    if i < 3:  # RGB channels
+                        if isinstance(v, int):
+                            normalized.append(v / 255 if v > 1 else v)
+                        else:
+                            normalized.append(float(v))
+                    else:  # Alpha channel
+                        normalized.append(float(v))
+                return tuple(normalized)
+        
+        return None
+    
+    # Convert input to RGBA
+    rgba = to_rgba(color_input)
+    if rgba is None:
+        return None
+    
+    # Extract components
+    components = []
+    for i, c in enumerate(rgba):
+        if i == 3 and not keep_alpha:
+            break
+        components.append(round(c * 255 if c <= 1 else c))
+    
+    # Determine if we can use short format
+    use_short = (allow_short and 
+                not force_long and 
+                len(components) in (3, 4) and
+                all((x % 17 == 0) for x in components[:3]))
+    
+    # Format the hex string
+    if use_short:
+        short_components = [x//17 for x in components[:3]] + components[3:]
+        hex_str = "".join(f"{x:1x}" for x in short_components)
+    else:
+        hex_str = "".join(f"{x:02x}" for x in components)
+    
+    # Apply case and prefix
+    if uppercase:
+        hex_str = hex_str.upper()
+    
+    return f"{prefix}{hex_str}"
 # ! format excel file
-
-
+ 
 def hex2argb(color):
     """
     Convert a color name or hex code to aARGB format required by openpyxl.
@@ -9753,6 +10033,105 @@ def copy_format(
         if "wb_target" in locals():
             wb_target.close()
 
+def set_sheet_visible(
+    fpath: str,
+    sheet_name: Union[int, str, None,list] = 1,
+    show: Union[bool, str] = True,
+    exclude: Union[List[str], None,list,int] = None,
+    verbose: bool = False
+) -> None:
+    """
+    Modify sheet visibility in an Excel workbook.
+    set_sheet_visible(fpath=dir_data_collection,sheet_name=None,show=1,verbose=1)
+    Args:
+        fpath (str): Path to the Excel workbook.
+        sheet_name (int | str | None): Index or name of the sheet to apply visibility to.
+                                       If None, all sheets are considered.
+        show (bool | str): Visibility mode. Can be:
+                           - True -> visible
+                           - False -> veryHidden
+                           - 'visible', 'hidden', 'veryHidden' as str
+        exclude (list[str] | None): List of sheet names to exclude from changes.
+        verbose (bool): If True, logs actions.
+    """
+
+    try: 
+        wb = fload(fpath, output="bit", get_validations=1)
+    except Exception as e:
+        raise FileNotFoundError(f"Unable to load workbook: {e}")
+
+    sheet_names = wb.sheetnames
+    if verbose:
+        print("Workbook loaded with sheets:")
+        for i, name in enumerate(sheet_names):
+            print(f"  [{i}] {name}")
+
+    excludes=[]
+    if exclude is None:
+        exclude=[]
+    if ~isinstance(exclude, list):
+        exclude = [exclude]
+    for exclude_ in exclude: 
+        if isinstance(exclude_, str):
+            excludes.append(strcmp(exclude_, sheet_names)[0])
+        elif isinstance(exclude_, int):
+            if 0 <= exclude_ < len(sheet_names):
+                excludes.append(sheet_names[exclude_])
+            else:
+                raise IndexError(f"sheet_name index {exclude_} is out of range:0~{len(sheet_names)-1}.")
+
+    # Resolve the sheet_name target
+    target_indices = []
+    if not isinstance(sheet_name,list):
+        sheet_name=[sheet_name]
+    for sheet_name_ in sheet_name:
+        if sheet_name_ is None:
+            target_indices = list(range(len(sheet_names)))
+            break
+        elif isinstance(sheet_name_, int):
+            if 0 <= sheet_name_ < len(sheet_names):
+                target_indices.append(sheet_name_)
+            else:
+                raise IndexError(f"sheet_name index {sheet_name_} is out of range :0~{len(sheet_names)-1}.")
+        elif isinstance(sheet_name_, str):
+            idx = strcmp(sheet_name_, sheet_names)[1]
+            if idx == -1:
+                raise ValueError(f"Sheet '{sheet_name_}' not found.")
+            target_indices.append(idx)
+
+    # Map show argument to valid state
+    valid_states = ["veryHidden", "visible", "hidden"]
+    if isinstance(show, str):
+        if show not in valid_states:
+            raise ValueError(f"Invalid show value '{show}'. Must be one of {valid_states}")
+        state = show
+    else:
+        state = "visible" if show else "veryHidden" 
+    # Modify sheet visibility
+    for idx in target_indices:
+        ws= wb[sheet_names[idx]]
+        if ws.title in excludes:
+            if verbose:
+                print(f"Skipping excluded sheet: '{ws.title}'")
+            continue
+        ws.sheet_state = state 
+    # Ensure at least one sheet is visible
+    visible_sheets = [s for s in wb.worksheets if s.sheet_state == "visible"]
+    not_visible_sheets = [s for s in wb.worksheets if s.sheet_state != "visible"]
+    if not visible_sheets:
+        fallback_sheet = wb.worksheets[0]
+        fallback_sheet.sheet_state = "visible"
+        if verbose:
+            print(f"No visible sheets found. Setting '{fallback_sheet.title}' to visible.")
+    if verbose:
+        print(f"visible sheets:{[s.title for s in visible_sheets]}")
+
+    try:
+        wb.save(fpath) 
+    except Exception as e:
+        raise IOError(f"Error saving workbook: {e}") 
+
+
 def format_excel(
     df: pd.DataFrame=None,
     filename:str=None,
@@ -14580,209 +14959,213 @@ def df_reducer(
 
 # example:
 # df_reducer(data=data_log, columns=markers, n_components=2)
+ 
 
 
-def get_df_format(data, threshold_unique=0.5, verbose=False):
+def get_df_format(data, threshold_unique=0.5, verbose=False, sample_size=1000):
     """
-    检测表格: long, wide or uncertain.
-
+    Detect whether a DataFrame is in long or wide format with optimized performance and accuracy.
+    
     Parameters:
-    - data (pd.DataFrame): DataFrame to check.
-    - threshold_unique (float): Proportion threshold for detecting categorical columns.
-
+    - data (pd.DataFrame): DataFrame to analyze
+    - threshold_unique (float): Threshold for categorical column detection (0-1)
+    - verbose (bool): Whether to print diagnostic messages
+    - sample_size (int): Maximum number of rows/columns to sample for large datasets
+    
     Returns:
-    - "long" if detected as long format,
+    - "long" if detected as long format
     - "wide" if detected as wide format
-    - "uncertain" if ambiguous.
+    - "uncertain" if format is ambiguous
     """
+    import pandas as pd
+    import numpy as np
     from scipy.stats import entropy
     from sklearn.cluster import AgglomerativeClustering
     from sklearn.preprocessing import StandardScaler
-
-    long_score,wide_score,fs = 0,0,500
-    n_rows, n_cols = data.shape
-    # -----to reduce memory, only check 500 rows/columns----
-    if n_rows > fs:
-        if verbose:
-            print(f"Sampling {fs} rows from {n_rows} rows.")
-        data = data.sample(n=fs, random_state=1)
-    if n_cols > fs:
-        if verbose:
-            print(f"Using first {fs} columns out of {n_cols} columns.")
-        data = data.iloc[:, :fs]
+    from sklearn.metrics import pairwise_distances
+    from collections import Counter
+    import re
+    # ----- Initial Setup and Sampling -----
     n_rows, n_cols = data.shape
+    if verbose:
+        print(f"Initial shape: {n_rows} rows, {n_cols} columns")
     
-    # Step 1: Row-Column Ratio Heuristic
-    if n_rows > 3 * n_cols:
-        long_score += 2
-        if verbose:
-            print(
-                "Row-Column Ratio suggests long format (many rows relative to columns)."
-            )
-    elif n_cols > 3 * n_rows:
-        wide_score += 2
-        if verbose:
-            print(
-                "Row-Column Ratio suggests wide format (many columns relative to rows)."
-            )
-
-    # Step 2: Unique-to-duplicate ratio and entropy for categorical variables
-    unique_counts = data.apply(lambda x: x.nunique())
+    # Sample data if too large
+    if n_rows > sample_size:
+        data = data.sample(n=sample_size, random_state=42)
+        n_rows = sample_size
+    if n_cols > sample_size:
+        data = data.iloc[:, :sample_size]
+        n_cols = sample_size
+    
+    # Early exit for tiny datasets
+    if n_rows < 3 or n_cols < 3:
+        return "uncertain"
+    
+    long_score = 0
+    wide_score = 0
+    
+    # ----- Feature Extraction -----
+    # Basic statistics
+    row_col_ratio = n_rows / n_cols if n_cols != 0 else float('inf')
+    
+    # Column types
+    numeric_cols = data.select_dtypes(include=np.number).columns
+    cat_cols = data.select_dtypes(include=['object', 'category']).columns
+    other_cols = [col for col in data.columns if col not in numeric_cols and col not in cat_cols]
+    
+    # Unique value analysis
+    unique_counts = data.nunique(dropna=False)
     duplicate_ratio = 1 - unique_counts / n_rows
-    if (duplicate_ratio > 0.2).sum() > 0.5 * n_cols:
-        wide_score += 2
-        if verbose:
-            print("High duplicate values in columns suggest wide format.")
-    else:
-        long_score += 1
-        if verbose:
-            print(
-                "Lower duplicate ratio suggests long format (higher row variability)."
-            )
-
-    # Calculate entropy for categorical columns
-    categorical_cols = data.select_dtypes(include=["object", "category"]).columns
-    if len(categorical_cols) > 0:
-        for col in categorical_cols:
-            counts = data[col].value_counts(normalize=True)
-            col_entropy = entropy(counts)
-            if col_entropy < 1.5:
-                long_score += 1
-                if verbose:
-                    print(
-                        f"Column '{col}' entropy suggests categorical, supporting long format."
-                    )
-            else:
-                wide_score += 1
-                if verbose:
-                    print(f"Column '{col}' entropy is higher, supporting wide format.")
-
-    # Step 3: Column grouping analysis for patterns in suffixes/prefixes
+    
+    # Missing values
+    missing_per_row = data.isna().sum(axis=1)
+    missing_per_col = data.isna().sum()
+    
+    # Column name patterns
     col_names = data.columns.astype(str)
-    suffix_count = sum("_" in col or col[-1].isdigit() for col in col_names)
-    if suffix_count > 0.3 * n_cols:
+    has_suffix = sum(bool(re.search(r'(_\d+|\d+_?$)', col)) for col in col_names)
+    has_time = sum(bool(re.search(r'(^time|^date|^year|^month|^day|^t\d+)', col.lower())) for col in col_names)
+    
+    # ----- Scoring Rules -----
+    
+    # 1. Row-Column Ratio (weighted)
+    if row_col_ratio > 5:
+        long_score += 3
+        if verbose: print(f"High row/col ratio ({row_col_ratio:.1f}) → long +3")
+    elif row_col_ratio < 0.2:
+        wide_score += 3
+        if verbose: print(f"Low row/col ratio ({row_col_ratio:.1f}) → wide +3")
+    elif row_col_ratio > 2:
+        long_score += 1
+        if verbose: print(f"Moderate row/col ratio ({row_col_ratio:.1f}) → long +1")
+    elif row_col_ratio < 0.5:
+        wide_score += 1
+        if verbose: print(f"Moderate row/col ratio ({row_col_ratio:.1f}) → wide +1")
+    
+    # 2. Duplication Patterns
+    high_dupe_cols = sum(duplicate_ratio > 0.3)
+    if high_dupe_cols > 0.6 * n_cols:
         wide_score += 2
-        if verbose:
-            print(
-                "Detected suffix/prefix patterns in column names, suggesting wide format."
-            )
-
-    # Step 4: Entity identifier detection for long format with categorical columns
-    if len(categorical_cols) > 0 and n_rows > n_cols:
-        entity_identifier_count = sum(
-            data.duplicated(subset=categorical_cols, keep=False)
-        )
-        if entity_identifier_count > 0.2 * n_rows:
+        if verbose: print(f"Many columns ({high_dupe_cols}/{n_cols}) with duplicates → wide +2")
+    elif high_dupe_cols < 0.2 * n_cols:
+        long_score += 1
+        if verbose: print(f"Few columns ({high_dupe_cols}/{n_cols}) with duplicates → long +1")
+    
+    # 3. Categorical Column Analysis
+    if len(cat_cols) > 0:
+        # Entropy analysis
+        cat_entropies = []
+        for col in cat_cols:
+            counts = data[col].value_counts(normalize=True, dropna=False)
+            cat_entropies.append(entropy(counts))
+        
+        avg_cat_entropy = np.mean(cat_entropies) if cat_entropies else 0
+        if avg_cat_entropy < 1.2:
             long_score += 2
-            if verbose:
-                print(
-                    "Significant duplicate rows based on categorical columns, suggesting long format."
-                )
-
-    # Step 5: Clustering analysis on numerical columns for correlation in wide format
-    numeric_cols = data.select_dtypes(include="number").columns
-    if len(numeric_cols) > 1:
-        try:
-            scaled_data = StandardScaler().fit_transform(data[numeric_cols].dropna())
-            clustering = AgglomerativeClustering(n_clusters=2).fit(scaled_data.T)
-            cluster_labels = pd.Series(clustering.labels_)
-            if cluster_labels.nunique() < len(numeric_cols) * 0.5:
-                wide_score += 2
-                if verbose:
-                    print(
-                        "Clustering on columns shows grouping, suggesting wide format."
-                    )
-        except Exception as e:
-            print(e) if verbose else None
-
-    # Step 6: Inter-column correlation analysis
-    if len(numeric_cols) > 1:
+            if verbose: print(f"Low categorical entropy ({avg_cat_entropy:.2f}) → long +2")
+        elif avg_cat_entropy > 2:
+            wide_score += 1
+            if verbose: print(f"High categorical entropy ({avg_cat_entropy:.2f}) → wide +1")
+        
+        # Entity identifier detection
+        if len(cat_cols) >= 2 and n_rows > 10:
+            dup_rows = data.duplicated(subset=cat_cols.tolist()[:2], keep=False).sum()
+            if dup_rows > 0.3 * n_rows:
+                long_score += 2
+                if verbose: print(f"Duplicate rows in categorical cols ({dup_rows}/{n_rows}) → long +2")
+    
+    # 4. Column Name Patterns
+    if has_suffix > 0.4 * n_cols:
+        wide_score += 2
+        if verbose: print(f"Many suffix patterns ({has_suffix}/{n_cols}) → wide +2")
+    if has_time > 0.3 * n_cols:
+        wide_score += 1
+        if verbose: print(f"Time-like columns ({has_time}/{n_cols}) → wide +1")
+    
+    # 5. Numeric Column Analysis (only if enough numeric columns)
+    if len(numeric_cols) > 2:
+        # Correlation analysis
         corr_matrix = data[numeric_cols].corr().abs()
-        avg_corr = (
-            corr_matrix.where(~np.eye(len(corr_matrix), dtype=bool)).mean().mean()
-        )
-        if avg_corr > 0.6:
+        avg_corr = corr_matrix.values[np.triu_indices_from(corr_matrix, k=1)].mean()
+        
+        if avg_corr > 0.5:
             wide_score += 2
-            if verbose:
-                print("High inter-column correlation suggests wide format.")
-
-    # Step 7: Missing value pattern analysis
-    missing_patterns = data.isna().sum(axis=1)
-    if missing_patterns.std() < 2:
+            if verbose: print(f"High numeric correlation ({avg_corr:.2f}) → wide +2")
+        elif avg_corr < 0.2:
+            long_score += 1
+            if verbose: print(f"Low numeric correlation ({avg_corr:.2f}) → long +1")
+        
+        # Entropy analysis
+        try:
+            numeric_data = data[numeric_cols].dropna()
+            if len(numeric_data) > 10:
+                numeric_entropy = numeric_data.apply(lambda x: entropy(pd.cut(x, bins=min(10, len(x.unique())).value_counts(normalize=True))))
+                if numeric_entropy.mean() < 1.5:
+                    wide_score += 1
+                    if verbose: print(f"Low numeric entropy ({numeric_entropy.mean():.2f}) → wide +1")
+        except Exception as e:
+            if verbose: print(f"Numeric entropy failed: {str(e)}")
+    
+    # 6. Missing Value Patterns
+    missing_row_std = missing_per_row.std()
+    if missing_row_std < 1 and missing_per_row.mean() > 0.1 * n_cols:
         wide_score += 1
-        if verbose:
-            print(
-                "Low variation in missing patterns across rows, supporting wide format."
-            )
-    elif missing_patterns.mean() < 1:
+        if verbose: print(f"Uniform missing pattern (std={missing_row_std:.2f}) → wide +1")
+    elif missing_per_row.mean() < 0.05 * n_cols:
         long_score += 1
-        if verbose:
-            print("Lower missing pattern suggests long format (less structured).")
-
-    # Step 8: Multi-level clustering on rows to detect block structure for wide format
-    if len(numeric_cols) > 1 and n_rows > 5:
+        if verbose: print(f"Few missing values → long +1")
+    
+    # 7. Advanced Clustering (only for medium/large datasets)
+    if len(numeric_cols) > 3 and n_rows > 10 and n_cols > 5:
         try:
-            clustering_rows = AgglomerativeClustering(n_clusters=2).fit(scaled_data)
-            if pd.Series(clustering_rows.labels_).nunique() < 2:
-                wide_score += 2
-                if verbose:
-                    print("Row clustering reveals homogeneity, suggesting wide format.")
+            # Efficient clustering with sampling
+            sample_data = data[numeric_cols].sample(n=min(100, n_rows), random_state=42)
+            scaled_data = StandardScaler().fit_transform(sample_data.dropna())
+            
+            if scaled_data.shape[0] > 5:
+                # Column clustering
+                col_dist = pairwise_distances(scaled_data.T)
+                col_clusters = AgglomerativeClustering(n_clusters=2, 
+                                                     affinity='precomputed', 
+                                                     linkage='complete').fit(col_dist)
+                cluster_counts = Counter(col_clusters.labels_)
+                if max(cluster_counts.values()) > 0.7 * len(numeric_cols):
+                    wide_score += 2
+                    if verbose: print(f"Column clustering shows dominant group → wide +2")
+                
+                # Row clustering
+                row_clusters = AgglomerativeClustering(n_clusters=2).fit(scaled_data)
+                row_cluster_counts = Counter(row_clusters.labels_)
+                if max(row_cluster_counts.values()) > 0.8 * scaled_data.shape[0]:
+                    wide_score += 1
+                    if verbose: print(f"Row clustering shows homogeneity → wide +1")
         except Exception as e:
-            print(e) if verbose else None
-
-    # Step 9: Sequential name detection for time-series pattern in wide format
-    if any(col.isdigit() or col.startswith("T") for col in col_names):
-        wide_score += 1
-        if verbose:
-            print("Detected time-like sequential column names, supporting wide format.")
-
-    # Step 10: Entropy of numeric columns
-    try:
-        numeric_entropy = data[numeric_cols].apply(
-            lambda x: entropy(pd.cut(x, bins=10).value_counts(normalize=True))
-        )
-        if numeric_entropy.mean() < 2:
-            wide_score += 2
-            if verbose:
-                print(
-                    "Low entropy in numeric columns indicates stability across columns, supporting wide format."
-                )
-    except Exception as e:
-        print(e) if verbose else None
-
-    # Step 11: Tie-breaking strategy if scores are equal
-    if wide_score == long_score:
-        if n_cols > n_rows:
-            wide_score += 1
-            if verbose:
-                print(
-                    "Tie-breaking based on column-major structure, favoring wide format."
-                )
-        elif n_rows > n_cols:
-            long_score += 1
-            if verbose:
-                print(
-                    "Tie-breaking based on row-major structure, favoring long format."
-                )
-        else:
-            if verbose:
-                print("Tie-breaking inconclusive; returning 'uncertain'.")
-            return "uncertain"
-
-    # Final decision
-    if wide_score > long_score:
-        if verbose:
-            print("Final decision: Wide format.")
-        return "wide"
-    elif long_score > wide_score:
-        if verbose:
-            print("Final decision: Long format.")
-        return "long"
+            if verbose: print(f"Clustering skipped: {str(e)}")
+    
+    # ----- Decision Logic -----
+    score_diff = long_score - wide_score
+    abs_diff = abs(score_diff)
+    
+    if verbose:
+        print(f"\nFinal scores - Long: {long_score}, Wide: {wide_score}")
+    
+    if abs_diff >= 3:
+        return "long" if score_diff > 0 else "wide"
+    elif abs_diff >= 1:
+        # Additional tie-breakers
+        if score_diff == 0:
+            if row_col_ratio > 1.5:
+                return "long"
+            elif row_col_ratio < 0.67:
+                return "wide"
+            elif len(cat_cols) > len(numeric_cols):
+                return "long"
+            else:
+                return "wide"
+        return "long" if score_diff > 0 else "wide"
     else:
-        if verbose:
-            print("Final decision: Uncertain format.")
         return "uncertain"
-
 #! ========== workbook, worksheet, wb,ws =============
 
 import openpyxl
@@ -17221,3 +17604,290 @@ def set_theme(
         color_codes=color_codes,
         rc=rc_params,
     )
+
+
+
+def df_wide_long(df):
+    rows, columns = df.shape
+    if columns > rows:
+        return "Wide"
+    elif rows > columns:
+        return "Long"
+
+def df2array(data: pd.DataFrame, x=None, y=None, hue=None, sort=False):
+    
+    def sort_rows_move_nan(arr, sort=False):
+        # Handle edge cases where all values are NaN
+        if np.all(np.isnan(arr)):
+            return arr  # Return unchanged if the entire array is NaN
+
+        if sort:
+            # Replace NaNs with a temporary large value for sorting
+            temp_value = (
+                np.nanmax(arr[np.isfinite(arr)]) + 1 if np.any(np.isfinite(arr)) else np.inf
+            )
+            arr_no_nan = np.where(np.isnan(arr), temp_value, arr)
+
+            # Sort each row
+            sorted_arr = np.sort(arr_no_nan, axis=1)
+
+            # Move NaNs to the end
+            result_arr = np.where(sorted_arr == temp_value, np.nan, sorted_arr)
+        else:
+            result_rows = []
+            for row in arr:
+                # Separate non-NaN and NaN values
+                non_nan_values = row[~np.isnan(row)]
+                nan_count = np.isnan(row).sum()
+                # Create a new row with non-NaN values followed by NaNs
+                new_row = np.concatenate([non_nan_values, [np.nan] * nan_count])
+                result_rows.append(new_row)
+            # Convert the list of rows back into a 2D NumPy array
+            result_arr = np.array(result_rows)
+
+        # Remove rows/columns that contain only NaNs
+        clean_arr = result_arr[~np.isnan(result_arr).all(axis=1)]
+        clean_arr_ = clean_arr[:, ~np.isnan(clean_arr).all(axis=0)]
+
+        return clean_arr_
+    # data = data.copy()
+    # data[y] = pd.to_numeric(data[y], errors="coerce")
+    # data = data.dropna(subset=[y])
+    if hue is None:
+        a = []
+        if sort:
+            cat_x = np.sort(data[x].unique().tolist()).tolist()
+        else:
+            cat_x = data[x].unique().tolist()
+        for i, x_ in enumerate(cat_x):
+            new_ = data.loc[data[x] == x_, y].to_list()
+            a = padcat(a, new_, axis=0)
+        return sort_rows_move_nan(a).T
+    else:
+        a = []
+        if sort:
+            cat_x = np.sort(data[x].unique().tolist()).tolist()
+            cat_hue = np.sort(data[hue].unique().tolist()).tolist()
+        else:
+            cat_x = data[x].unique().tolist()
+            cat_hue = data[hue].unique().tolist()
+        for i, x_ in enumerate(cat_x):
+            for j, hue_ in enumerate(cat_hue):
+                new_ = data.loc[(data[x] == x_) & (data[hue] == hue_), y].to_list()
+                a = padcat(a, new_, axis=0)
+        return sort_rows_move_nan(a).T
+
+
+def array2df(data: np.ndarray):
+    df = pd.DataFrame()
+    df["group"] = (
+        np.tile(
+            ["group" + str(i) for i in range(1, data.shape[1] + 1)], [data.shape[0], 1]
+        )
+        .reshape(-1, 1, order="F")[:, 0]
+        .tolist()
+    )
+    df["value"] = data.reshape(-1, 1, order="F")
+    return df
+
+
+def padcat(*args, fill_value=np.nan, axis=1, order="row"):
+    """
+    Concatenate vectors with padding.
+
+    Parameters:
+    *args : variable number of list or 1D arrays
+        Input arrays to concatenate.
+    fill_value : scalar, optional
+        The value to use for padding the shorter lists (default is np.nan).
+    axis : int, optional
+        The axis along which to concatenate (0 for rows, 1 for columns, default is 1).
+    order : str, optional
+        The order for flattening when required: "row" or "column" (default is "row").
+
+    Returns:
+    np.ndarray
+        A 2D array with the input arrays concatenated along the specified axis,
+        padded with fill_value where necessary.
+
+
+# Example usage:
+a = [1, np.nan]
+b = [1, 3, 4, np.nan, 2, np.nan]
+c = [1, 2, 3, 4, 5, 6, 7, 8, 10]
+d = padcat(a, b)
+result1 = padcat(d, c)
+result2 = padcat(a, b, c)
+print("Result of padcat(d, c):\n", result1)
+print("Result of padcat(a, b, c):\n", result2)
+    """
+    # Set the order for processing
+    if "ro" in order.lower():
+        order = "C"  # row-major order
+    else:
+        order = "F"  # column-major order
+
+    # Process input arrays based on their dimensions
+    processed_arrays = []
+    for arg in args:
+        arr = np.asarray(arg)
+        if arr.ndim == 1:
+            processed_arrays.append(arr)  # Keep 1D arrays as is
+        elif arr.ndim == 2:
+            if axis == 0:
+                # If concatenating along rows, split 2D arrays into 1D arrays row-wise
+                processed_arrays.extend(arr)
+            elif axis == 1:
+                # If concatenating along columns, split 2D arrays into 1D arrays column-wise
+                processed_arrays.extend(arr.T)
+            else:
+                raise ValueError("axis must be 0 or 1")
+        else:
+            raise ValueError("Input arrays must be 1D or 2D")
+
+    if axis == 0:
+        # Concatenate along rows
+        max_len = max(arr.size for arr in processed_arrays)
+        result = np.full((len(processed_arrays), max_len), fill_value)
+        for i, arr in enumerate(processed_arrays):
+            result[i, : arr.size] = arr
+    elif axis == 1:
+        # Concatenate along columns
+        max_len = max(arr.size for arr in processed_arrays)
+        result = np.full((max_len, len(processed_arrays)), fill_value)
+        for i, arr in enumerate(processed_arrays):
+            result[: arr.size, i] = arr
+    else:
+        raise ValueError("axis must be 0 or 1")
+
+    return result
+
+ 
+# ========== memory cleaner ==========
+import gc
+import os
+import sys
+import psutil
+import platform
+import ctypes
+import subprocess
+import warnings 
+import time
+
+class MemoryOptimizer:
+    def __init__(self, verbose: bool = True, aggressive_mode: bool = True):
+        self.verbose = verbose
+        self.aggressive_mode = aggressive_mode
+        self.system = platform.system()
+        self.process = psutil.Process(os.getpid())
+        self.start_time = time.time()
+        self.memory_history = []
+
+    def log(self, msg: str, level: str = "INFO"):
+        if self.verbose:
+            rss = self.process.memory_info().rss / (1024 ** 2)
+            elapsed = time.time() - self.start_time
+            print(f"[{level}][{elapsed:.2f}s][{rss:.1f}MB] {msg}")
+
+    def collect_garbage(self):
+        self.log("Performing deep garbage collection...")
+        stats = {}
+        before_mem = self.process.memory_info().rss
+        for gen in reversed(range(3)):
+            collected = gc.collect(gen)
+            self.log(f"GC Gen {gen}: Collected {collected}")
+        gc.garbage.clear()
+        after_mem = self.process.memory_info().rss
+        stats['freed_mb'] = (before_mem - after_mem) / (1024 ** 2)
+        return stats
+
+    def clear_frameworks(self):
+        result = {}
+        try:
+            import torch
+            if torch.cuda.is_available():
+                self.log("Clearing PyTorch cache...")
+                torch.cuda.empty_cache()
+                torch.cuda.ipc_collect()
+                result['pytorch'] = 'cleared'
+        except Exception as e:
+            self.log(f"PyTorch skipped: {e}", "WARNING")
+
+        try:
+            import tensorflow as tf
+            self.log("Clearing TensorFlow session...")
+            tf.keras.backend.clear_session()
+            result['tensorflow'] = 'cleared'
+        except Exception as e:
+            self.log(f"TensorFlow skipped: {e}", "WARNING")
+
+        try:
+            import cv2
+            self.log("Closing OpenCV windows...")
+            cv2.destroyAllWindows()
+            result['opencv'] = 'cleared'
+        except Exception:
+            pass
+
+        try:
+            import matplotlib.pyplot as plt
+            self.log("Closing matplotlib figures...")
+            plt.close('all')
+            result['matplotlib'] = 'cleared'
+        except Exception:
+            pass
+
+        return result
+
+    def clear_system_caches(self):
+        result = {}
+        self.log("Attempting full system cache clearance...")
+        try:
+            if self.system == "Linux":
+                subprocess.run(["sync"], check=True)
+                subprocess.run(["sudo", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"], check=True)
+                result['linux'] = 'caches dropped'
+            elif self.system == "Darwin":
+                subprocess.run(["sudo", "purge"], check=True)
+                result['macos'] = 'purge run'
+            elif self.system == "Windows":
+                ctypes.windll.psapi.EmptyWorkingSet(-1)
+                if self.aggressive_mode:
+                    ctypes.windll.kernel32.SetProcessWorkingSetSizeEx(
+                        -1, ctypes.c_size_t(-1), ctypes.c_size_t(-1), ctypes.c_uint(0x1)
+                    )
+                result['windows'] = 'working set emptied'
+        except Exception as e:
+            self.log(f"System cache clearing failed: {e}", "ERROR")
+        return result
+
+    def profile(self) -> Dict[str, Any]:
+        mem = self.process.memory_info()
+        vm = psutil.virtual_memory()
+        profile = {
+            'rss_mb': mem.rss / (1024 ** 2),
+            'vms_mb': mem.vms / (1024 ** 2),
+            'used_gb': vm.used / (1024 ** 3),
+            'available_gb': vm.available / (1024 ** 3),
+            'percent': vm.percent,
+        }
+        self.memory_history.append(profile)
+        return profile
+
+    def optimize(self) -> Dict[str, Any]:
+        result = {}
+        result['before'] = self.profile()
+        result['gc'] = self.collect_garbage()
+        result['frameworks'] = self.clear_frameworks()
+        result['system'] = self.clear_system_caches()
+        result['after'] = self.profile()
+        saved = result['before']['rss_mb'] - result['after']['rss_mb']
+        result['saved_mb'] = saved
+        result['saved_percent'] = (saved / result['before']['rss_mb']) * 100 if result['before']['rss_mb'] else 0
+        self.log(f"Optimization complete: Saved {saved:.2f} MB ({result['saved_percent']:.1f}%)", "SUCCESS")
+        return result
+
+
+def cleaner(verbose: bool = True, aggressive: bool = True) -> Dict[str, Any]:
+    optimizer = MemoryOptimizer(verbose=verbose, aggressive_mode=aggressive)
+    return optimizer.optimize()