py2ls 0.2.4.6__tar.gz → 0.2.4.7__tar.gz

{py2ls-0.2.4.6 → py2ls-0.2.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.2.4.6
+Version: 0.2.4.7
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com

{py2ls-0.2.4.6 → py2ls-0.2.4.7}/py2ls/bio.py

@@ -166,9 +166,23 @@ def get_probe(
     if platform_id is None:
         df_meta = get_meta(geo=geo, dataset=dataset, verbose=False)
         platform_id = df_meta["platform_id"].unique().tolist()
-        platform_id = platform_id[0] if len(platform_id) == 1 else platform_id
         print(f"Platform: {platform_id}")
-    df_probe = geo[dataset].gpls[platform_id].table
+    if len(platform_id) > 1:
+        df_probe= geo[dataset].gpls[platform_id[0]].table
+        # df_probe=pd.DataFrame()
+        # # Iterate over each platform ID and collect the probe tables
+        # for platform_id_ in platform_id:
+        #     if platform_id_ in geo[dataset].gpls:
+        #         df_probe_ = geo[dataset].gpls[platform_id_].table
+        #         if not df_probe_.empty:
+        #             df_probe=pd.concat([df_probe, df_probe_])
+        #         else:
+        #             print(f"Warning: Probe table for platform {platform_id_} is empty.")
+        #     else:
+        #         print(f"Warning: Platform ID {platform_id_} not found in dataset {dataset}.")
+    else:
+        df_probe= geo[dataset].gpls[platform_id[0]].table
+    
     if df_probe.empty:
         print(
             f"Warning: cannot find the probe info. 看一下是不是在单独的文件中包含了probe信息"
@@ -215,9 +229,12 @@ def get_data(geo: dict, dataset: str = "GSE25097", verbose=False):
     df_expression = get_expression_data(geo, dataset=dataset)
     if not df_expression.select_dtypes(include=["number"]).empty:
         # 如果数据全部是counts类型的话, 则使用TMM进行normalize
-        if 'counts' in get_data_type(df_expression): 
-            print(f"{dataset}'s type is raw read counts, nomalized(transformed) via 'TMM'")
-            df_expression=counts2expression(df_expression.T).T 
+        if 'counts' in get_data_type(df_expression):
+            try:
+                df_expression=counts2expression(df_expression.T).T 
+                print(f"{dataset}'s type is raw read counts, nomalized(transformed) via 'TMM'")
+            except Exception as e: 
+                print("raw counts data")
     if any([df_probe.empty, df_expression.empty]):
         print(
             f"got empty values, check the probe info. 看一下是不是在单独的文件中包含了probe信息"

{py2ls-0.2.4.6 → py2ls-0.2.4.7}/py2ls/ips.py

@@ -555,14 +555,28 @@ def shared(*args, strict=True, n_shared=2, verbose=True):
         # Get elements that appear in at least n_shared lists
         shared_elements = [item for item, count in element_count.items() if count >= n_shared]
 
-    shared_elements = flatten(shared_elements, verbose=verbose) 
+    shared_elements = flatten(shared_elements, verbose=verbose)
     if verbose:
         elements2show = shared_elements if len(shared_elements)<10 else shared_elements[:5]
         print(f"{' '*2}{len(shared_elements)} elements shared: {' '*2}{elements2show}")
         print("********* checking shared elements *********")
     return shared_elements
 
-def flatten(nested: Any, unique_list=True, verbose=True):
+def not_shared(*args, strict=True, n_shared=2, verbose=False):
+    """
+    To find the elements in list1 that are not shared with list2 while maintaining the original order of list1
+    usage:
+        list1 = [1, 8, 3, 3, 4, 5]
+        list2 = [4, 5, 6, 7, 8]
+        not_shared(list1,list2)# output [1,3]
+    """
+    _common = shared(*args, strict=strict, n_shared=n_shared, verbose=verbose)
+    list1 = args[0]
+    _not_shared=[item for item in list1 if item not in _common]
+    return flatten(_not_shared, verbose=verbose)
+
+
+def flatten(nested: Any, unique_list=True, verbose=False):
     """
     Recursively flattens a nested structure (lists, tuples, dictionaries, sets) into a single list.
     Parameters:
@@ -1560,13 +1574,26 @@ def unzip(dir_path, output_dir=None):
             tar_ref.extractall(output_dir)
         return output_dir
     # Handle .gz files
-    if dir_path.endswith(".gz"):
+    if dir_path.endswith(".gz") or dir_path.endswith(".gzip"):
         import gzip
 
         output_file = os.path.splitext(dir_path)[0]  # remove the .gz extension
-        with gzip.open(dir_path, "rb") as gz_file:
-            with open(output_file, "wb") as out_file:
-                shutil.copyfileobj(gz_file, out_file)
+        try:
+            with gzip.open(dir_path, "rb") as gz_file:
+                with open(output_file, "wb") as out_file:
+                    shutil.copyfileobj(gz_file, out_file)
+            print(f"unzipped '{dir_path}' to '{output_file}'")
+        except FileNotFoundError:
+            print(f"Error: The file '{dir_path}' was not found.")
+        except PermissionError:
+            print(f"Error: Permission denied when accessing '{dir_path}' or writing to '{output_file}'.")
+        except Exception as e:
+            try:
+                import tarfile
+                with tarfile.open(dir_path, 'r:gz') as tar:
+                    tar.extractall(path=output_file)
+            except Exception as final_e:
+                print(f"An final unexpected error occurred: {final_e}")
         return output_file
 
     # Handle .zip files
@@ -1648,9 +1675,12 @@ def is_df_abnormal(df: pd.DataFrame, verbose=False) -> bool:
     False: normal
 
     """
+    if not isinstance(df, pd.DataFrame):
+        return False
+    df.columns = df.columns.astype(str)# 把它变成str, 这样就可以进行counts运算了
     # Initialize a list to hold messages about abnormalities
     messages = []
-    is_abnormal = False
+    is_abnormal = True
     # Check the shape of the DataFrame
     actual_shape = df.shape
     messages.append(f"Shape of DataFrame: {actual_shape}")
@@ -1745,10 +1775,12 @@ def fload(fpath, kind=None, **kwargs):
             content = file.read()
         return content
 
-    def load_html(fpath):
-        with open(fpath, "r") as file:
-            content = file.read()
-        return content
+    # def load_html(fpath):
+    #     with open(fpath, "r") as file:
+    #         content = file.read()
+    #     return content
+    def load_html(fpath,**kwargs):
+        return pd.read_html(fpath,**kwargs)
 
     def load_json(fpath, **kwargs):
         output=kwargs.pop("output","json")
@@ -1962,8 +1994,8 @@ def fload(fpath, kind=None, **kwargs):
                                 # display(df.head(2))
                                 # print(f"is_df_abnormal:{is_df_abnormal(df, verbose=0)}")
                                 if not is_df_abnormal(df, verbose=0):
-                                    display(df.head(2))
-                                    print(f"shape: {df.shape}")
+                                    display(df.head(2)) if isinstance(df, pd.DataFrame) else display("it is not a DataFrame")
+                                    print(f"shape: {df.shape}") if isinstance(df, pd.DataFrame) else display("it is not a DataFrame")
                                     return df
                             except EmptyDataError as e:
                                 continue
@@ -1987,6 +2019,42 @@ def fload(fpath, kind=None, **kwargs):
             pass
         return df
 
+
+    def load_parquet(fpath, **kwargs):
+        """
+        Load a Parquet file into a Pandas DataFrame with advanced options.
+
+        Parameters:
+        - fpath (str): The file path to the Parquet file.
+        - engine (str): The engine to use for reading the Parquet file (default is 'pyarrow').
+        - columns (list): List of columns to load. If None, loads all columns.
+        - verbose (bool): If True, prints additional information about the loading process.
+        - filters (list): List of filter conditions for predicate pushdown.
+        - **kwargs: Additional keyword arguments for `pd.read_parquet`.
+
+        Returns:
+        - df (DataFrame): The loaded DataFrame.
+        """
+        
+        engine = kwargs.get("engine", "pyarrow")
+        verbose = kwargs.pop("verbose", False)
+        
+        if verbose:
+            use_pd("read_parquet", verbose=verbose)
+        try:
+            df = pd.read_parquet(fpath, engine=engine, **kwargs)
+            if verbose:
+                if 'columns' in kwargs:
+                    print(f"Loaded columns: {kwargs['columns']}")
+                else:
+                    print("Loaded all columns.")
+            print(f"shape: {df.shape}")
+        except Exception as e:
+            print(f"An error occurred while loading the Parquet file: {e}")
+            df = None
+
+        return df 
+
     def load_ipynb(fpath, **kwargs):
         as_version = kwargs.get("as_version", 4)
         with open(fpath, "r") as file:
@@ -2055,51 +2123,21 @@ def fload(fpath, kind=None, **kwargs):
         kind = kind.lower()
     kind = kind.lstrip(".").lower()
     img_types = [
-        "bmp",
-        "eps",
-        "gif",
-        "icns",
-        "ico",
-        "im",
-        "jpg",
-        "jpeg",
-        "jpeg2000",
-        "msp",
-        "pcx",
-        "png",
-        "ppm",
-        "sgi",
-        "spider",
-        "tga",
-        "tiff",
-        "tif",
-        "webp",
-        "json",
+        "bmp","eps","gif","png","jpg","jpeg","jpeg2000","tiff","tif",
+        "icns","ico","im","msp","pcx","ppm","sgi","spider","tga","webp",
     ]
     doc_types = [
-        "docx",
-        "txt",
-        "md",
-        "html",
-        "json",
-        "yaml",
-        "xml",
-        "csv",
-        "xlsx",
-        "pdf",
+        "docx","pdf",
+        "txt","csv","xlsx","tsv","parquet","snappy",
+        "md","html",
+        "json","yaml","xml",
         "ipynb",
+        "mtx"
     ]
     zip_types = [
-        "gz",
-        "zip",
-        "7z",
-        "tar",
-        "tar.gz",
-        "tar.bz2",
-        "bz2",
-        "xz",
-        "rar",
-        "tgz",
+        "gz","zip","7z","rar","tgz",
+        "tar","tar.gz","tar.bz2",
+        "bz2","xz","gzip"
     ]
     other_types = ["fcs"]
     supported_types = [*doc_types, *img_types, *zip_types, *other_types]
@@ -2128,14 +2166,14 @@ def fload(fpath, kind=None, **kwargs):
     elif kind == "txt" or kind == "md":
         return load_txt_md(fpath)
     elif kind == "html":
-        return load_html(fpath)
+        return load_html(fpath, **kwargs)
     elif kind == "json":
-        return load_json(fpath)
+        return load_json(fpath, **kwargs)
     elif kind == "yaml":
         return load_yaml(fpath)
     elif kind == "xml":
         return load_xml(fpath)
-    elif kind == "csv":
+    elif kind in ["csv","tsv"]:
         content = load_csv(fpath, **kwargs)
         return content
     elif kind in ["ods", "ods", "odt"]:
@@ -2146,14 +2184,25 @@ def fload(fpath, kind=None, **kwargs):
         engine = kwargs.get("engine", "xlrd")
         kwargs.pop("engine", None)
         content = load_excel(fpath, engine=engine, **kwargs)
+        print(f"shape: {content.shape}")
         display(content.head(3))
         return content
     elif kind == "xlsx":
         content = load_excel(fpath, **kwargs)
         display(content.head(3))
+        print(f"shape: {content.shape}")
+        return content
+    elif kind=='mtx':
+        from scipy.io import mmread
+        dat_mtx=mmread(fpath)
+        content=pd.DataFrame.sparse.from_spmatrix(dat_mtx,**kwargs)
+        display(content.head(3))
+        print(f"shape: {content.shape}")
         return content
     elif kind == "ipynb":
         return load_ipynb(fpath, **kwargs)
+    elif kind in ['parquet','snappy']:
+        return load_parquet(fpath,**kwargs)
     elif kind == "pdf":
         # print('usage:load_pdf(fpath, page="all", verbose=False)')
         return load_pdf(fpath, **kwargs)
@@ -2199,9 +2248,7 @@ def fload(fpath, kind=None, **kwargs):
         return meta, data
 
     else:
-        # try:
-        #     content = load_csv(fpath, **kwargs)
-        # except:
+        print("direct reading...")
         try:
             try:
                 with open(fpath, "r", encoding="utf-8") as f:
@@ -2501,6 +2548,25 @@ def fsave(
         tree = etree.ElementTree(root)
         tree.write(fpath, pretty_print=True, xml_declaration=True, encoding="UTF-8")
 
+    def save_parquet(fpath:str, data:pd.DataFrame, **kwargs): 
+        engine = kwargs.pop("engine","auto") # auto先试pyarrow, 不行就转为fastparquet, {‘auto’, ‘pyarrow’, ‘fastparquet’}
+        compression=kwargs.pop("compression",None) # Use None for no compression. Supported options: ‘snappy’, ‘gzip’, ‘brotli’, ‘lz4’, ‘zstd’
+        try:
+            # Attempt to save with "pyarrow" if engine is set to "auto"
+                data.to_parquet(fpath, engine=engine, compression=compression, **kwargs)
+                print(f"DataFrame successfully saved to {fpath} with engine '{engine}' and {compression} compression.") 
+        except Exception as e:
+            print(f"Error using with engine '{engine}' and {compression} compression: {e}")
+            if "Sparse" in str(e):
+                try:
+                    # Handle sparse data by converting columns to dense
+                    print("Attempting to convert sparse columns to dense format...")
+                    data = data.apply(lambda x: x.sparse.to_dense() if pd.api.types.is_sparse(x) else x)
+                    save_parquet(fpath, data=data,**kwargs)
+                except Exception as last_e:
+                    print(f"After converted sparse columns to dense format, Error using with engine '{engine}' and {compression} compression: {last_e}")
+                    
+
     if kind is None:
         _, kind = os.path.splitext(fpath)
         kind = kind.lower()
@@ -2546,6 +2612,15 @@ def fsave(
         save_yaml(fpath, content, **kwargs)
     elif kind == "ipynb":
         save_ipynb(fpath, content, **kwargs)
+    elif kind.lower() in ["parquet","pq","big","par"]:
+        compression=kwargs.pop("compression",None) # Use None for no compression. Supported options: ‘snappy’, ‘gzip’, ‘brotli’, ‘lz4’, ‘zstd’
+        # fix the fpath ends
+        if not '.parquet' in fpath:
+            fpath=fpath.replace(kind, 'parquet')
+        if compression is not None:
+            if not fpath.endswith(compression):
+                fpath=fpath+f".{compression}"
+        save_parquet(fpath=fpath, data=content,compression=compression,**kwargs)
     else:
         try:
             netfinder.downloader(url=content, dir_save=dirname(fpath), kind=kind)
@@ -5518,7 +5593,21 @@ def df_reducer(
 ) -> pd.DataFrame:
     """
     Reduces the dimensionality of the selected DataFrame using PCA or UMAP.
-
+    method: 
+        1. 'umap': 
+            - big dataset and global structure, often preferred in large-scale datasets for 
+            visualization and dimensionality reduction, balancing speed and quality of visualization.
+            - t-SNE excels at preserving local structure (i.e., clusters), but it often loses global 
+            relationships, causing clusters to appear in arbitrary proximities to each other.
+        2. 'pca': 
+            - t-SNE excels at preserving local structure (i.e., clusters), but it often loses global 
+                relationships, causing clusters to appear in arbitrary proximities to each other.
+            - useful as a preprocessing step and in datasets where linear relationships dominate.
+        3. 't-SNE': 
+            a. t-SNE excels at preserving local structure (i.e., clusters), but it often loses global 
+                relationships, causing clusters to appear in arbitrary proximities to each other.
+            b. often preferred in large-scale datasets for visualization and dimensionality 
+                reduction, balancing speed and quality of visualization.
     Parameters:
     -----------
     data : pd.DataFrame

{py2ls-0.2.4.6 → py2ls-0.2.4.7}/py2ls/plot.py

@@ -2337,40 +2337,20 @@ def get_color(
         if n == 1:
             colorlist = ["#3A4453"]
         elif n == 2:
-            colorlist = ["#3A4453", "#DF5932"]
+            colorlist = ["#3A4453", "#FBAF63"]
         elif n == 3:
-            colorlist = ["#3A4453", "#DF5932", "#299D8F"]
+            colorlist = ["#3A4453", "#FBAF63", "#299D8F"]
         elif n == 4:
-            # colorlist = ["#3A4453", "#DF5932", "#EBAA00", "#0B4083"]
-            colorlist = ["#81C6BD", "#FBAF63", "#F2675B", "#72A1C9"]
+            colorlist = ["#087cf7", "#FBAF63",  "#3C898A","#FF2C00"]
         elif n == 5:
-            colorlist = [
-                "#3A4453",
-                "#427AB2",
-                "#F09148",
-                "#DBDB8D",
-                "#C59D94",
-                "#AFC7E8",
-            ]
+            colorlist = ["#459AA9", "#B25E9D", "#4B8C3B","#EF8632","#FF2C00"]
         elif n == 6:
-            colorlist = [
-                "#3A4453",
-                "#427AB2",
-                "#F09148",
-                "#DBDB8D",
-                "#C59D94",
-                "#E53528",
-            ]
+            colorlist = ["#459AA9", "#B25E9D", "#4B8C3B","#EF8632", "#24578E","#FF2C00"]
+        elif n==7:
+            colorlist = [ "#7F7F7F", "#459AA9", "#B25E9D", "#4B8C3B","#EF8632", "#24578E" "#FF2C00"] 
         else:
-            colorlist = [
-                "#474747",
-                "#FF2C00",
-                "#0C5DA5",
-                "#845B97",
-                "#58BBCC",
-                "#FF9500",
-                "#D57DBE",
-            ]
+            colorlist = ['#FF7F0E','#2CA02C','#D62728','#9467BD','#E377C2','#7F7F7F','#7BB8CC','#06daf2']
+        
         by = "start"
     elif any(["cub" in cmap.lower(), "sns" in cmap.lower()]):
         if kwargs:
@@ -3106,9 +3086,9 @@ def plotxy(
             palette = kws_scatter.pop(
                 "palette",
                 (
-                    sns.color_palette("tab10", data[hue].nunique())
+                    sns.color_palette("tab20", data[hue].nunique())
                     if hue is not None
-                    else sns.color_palette("tab10")
+                    else sns.color_palette("tab20")
                 ),
             )
             s = kws_scatter.pop("s", 10)
@@ -3202,7 +3182,7 @@ def volcano(
     top_genes=[5, 5], # [down-regulated, up-regulated]
     thr_x=np.log2(1.5), # default: 0.585
     thr_y=-np.log10(0.05),
-    sort_xy="x", #'y'
+    sort_xy="x", #'y', 'xy'
     colors=("#00BFFF", "#9d9a9a", "#FF3030"),
     s=20,
     fill=True,  # plot filled scatter

{py2ls-0.2.4.6 → py2ls-0.2.4.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "py2ls"
-version = "0.2.4.6"
+version = "0.2.4.7"
 description = "py(thon)2(too)ls"
 authors = ["Jianfeng <Jianfeng.Liu0413@gmail.com>"]
 readme = "README.md"