halib 0.2.30__py3-none-any.whl

halib/filetype/yamlfile.py

@@ -0,0 +1,87 @@
+import time
+import networkx as nx
+from rich import inspect
+from rich.pretty import pprint
+from rich.console import Console
+
+from omegaconf import OmegaConf
+from argparse import ArgumentParser
+
+from ..system.path import *
+
+console = Console()
+
+
+def _load_yaml_recursively(
+    yaml_file, yaml_files=[], share_nx_graph=nx.DiGraph(), log_info=False
+):
+    conf = OmegaConf.load(yaml_file)
+    yaml_files.append(yaml_file)
+    if "__base__" in conf:
+        parent = conf["__base__"]
+        if isinstance(parent, str):
+            parent = [parent]
+        for p in parent:
+            edge = (yaml_file, p)
+            share_nx_graph.add_edge(*edge)
+            for cycle in nx.simple_cycles(share_nx_graph):
+                assert False, f"Cyclic dependency detected: {cycle}"
+            # update conf with parent; BY loading parent and merging with conf (the child)
+            conf = OmegaConf.merge(
+                _load_yaml_recursively(p, yaml_files, share_nx_graph), conf
+            )
+    if log_info:
+        console.rule()
+        console.print(f"current yaml_file: {yaml_file}")
+        inspect(yaml_files)
+        pprint(OmegaConf.to_container(conf, resolve=True))
+        time.sleep(1)
+    return conf
+
+
+def load_yaml(yaml_file, to_dict=False, log_info=False):
+    yaml_files = []
+    share_nx_graph = nx.DiGraph()
+    omgconf = _load_yaml_recursively(
+        yaml_file,
+        yaml_files=yaml_files,
+        share_nx_graph=share_nx_graph,
+        log_info=log_info,
+    )
+
+    if to_dict:
+        return OmegaConf.to_container(omgconf, resolve=True)
+    else:
+        return omgconf
+
+
+def load_yaml_with_PC_abbr(
+    yaml_file, abbr_disk_map=ABBR_DISK_MAP
+):
+    # load yaml file
+    data_dict = load_yaml(yaml_file=yaml_file, to_dict=True)
+    # Normalize paths in the loaded data
+    data_dict = normalize_paths(
+        data_dict, get_working_disk(abbr_disk_map), get_os_platform()
+    )
+    return data_dict
+
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument(
+        "-cfg", "--cfg", type=str, help="cfg file", default="cfg__default.yaml"
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    cfg_file = args.cfg
+    cfg = load_yaml(cfg_file, to_dict=True)
+    console.rule()
+    pprint(cfg)
+
+
+if __name__ == "__main__":
+    main()

halib/gdrive.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+
+"""
+    Upload folder to Google Drive
+"""
+import ast
+import os
+
+import googleapiclient.errors
+# Import Google libraries
+from pydrive.auth import GoogleAuth
+from pydrive.drive import GoogleDrive
+from pydrive.files import GoogleDriveFileList
+
+from halib.system import filesys
+from halib.filetype import textfile
+
+# Import general libraries
+
+ggDrive = None
+
+
+def get_gg_drive(settings_file='settings.yaml'):
+    """
+        Authenticate to Google API
+    """
+    global ggDrive
+    if ggDrive is None:
+        ggAuth = GoogleAuth(settings_file=settings_file)
+        ggDrive = GoogleDrive(ggAuth)
+    return ggDrive
+
+
+def get_folder_id(gg_parent_folder_id, folder_name):
+    """
+    Check if destination folder exists and return it's ID
+    """
+
+    # Auto-iterate through all files in the parent folder.
+    drive = get_gg_drive()
+    file_list = GoogleDriveFileList()
+    try:
+        file_list = drive.ListFile(
+            {'q': "'{0}' in parents and trashed=false".format(gg_parent_folder_id)}
+        ).GetList()
+    # Exit if the parent folder doesn't exist
+    except googleapiclient.errors.HttpError as err:
+        # Parse error message
+        message = ast.literal_eval(err.content)['error']['message']
+        if message == 'File not found: ':
+            print(message + folder_name)
+            exit(1)
+        # Exit with stacktrace in case of other error
+        else:
+            raise
+
+    # Find the the destination folder in the parent folder's files
+    for file1 in file_list:
+        if file1['title'] == folder_name:
+            print('title: %s, id: %s' % (file1['title'], file1['id']))
+            return file1['id']
+
+
+def create_folder(folder_name, gg_parent_folder_id):
+    """
+        Create folder on Google Drive
+    """
+
+    folder_metadata = {
+        'title': folder_name,
+        # Define the file type as folder
+        'mimeType': 'application/vnd.google-apps.folder',
+        # ID of the parent folder
+        'parents': [{"kind": "drive#fileLink", "id": gg_parent_folder_id}]
+    }
+    drive = get_gg_drive()
+    folder = drive.CreateFile(folder_metadata)
+    folder.Upload()
+
+    # Return folder information
+    # print('title: %s, id: %s' % (folder['title'], folder['id']))
+    return folder['id']
+
+
+def is_in_ignore_list(local_path, ignore_list=None):
+    in_ignore_list = False
+    if ignore_list:
+        for path in ignore_list:
+            if path in local_path:
+                in_ignore_list = True
+                break
+    return in_ignore_list
+
+
+def upload_file(local_file_path, gg_folder_id, ignore_list=None):
+    """
+        Upload local file to Google Drive folder
+    """
+    drive = get_gg_drive()
+    if not is_in_ignore_list(local_file_path, ignore_list):
+        print('uploading ' + local_file_path)
+        # Upload file to folder.
+        title = filesys.get_file_name(local_file_path, split_file_ext=False)
+
+        # delete file if exist on gg folder
+        query = f"'{gg_folder_id}'  in parents and trashed=false"
+        file_list = drive.ListFile({'q': f"{query}"}).GetList()
+        for file in file_list:
+            if file['title'] == title:
+                print(f'[DELETE] {title} on Google Drive')
+                file.Delete()
+                break
+
+        f = drive.CreateFile(
+            {"title": f"{title}",
+             "parents": [{"kind": "drive#fileLink", "id": gg_folder_id}]})
+        f.SetContentFile(local_file_path)
+        f.Upload()
+    # Skip the file if it's empty
+    else:
+        print('file {0} is empty or in ignore list'.format(local_file_path))
+
+
+def recursive_walk_and_upload(local_folder_path, gg_folder_id,
+                              processed_path, ignore_list=None):
+    for root, sub_folders, files in os.walk(local_folder_path):
+        # already processed folder
+        if root in processed_path:
+            print(f'[SKIP] already processed folder {root}')
+            return
+        print(f'\n\n[RECURSIVE] {local_folder_path}, {gg_folder_id}')
+        print(f'[FF] {root} {sub_folders} {files}')
+        if sub_folders:
+            for sub_folder in sub_folders:
+                sub_folder_path = os.path.join(root, sub_folder)
+                print(f'process {sub_folder_path}')
+                if is_in_ignore_list(sub_folder_path, ignore_list):
+                    continue
+                # Get destination folder ID
+                gg_sub_folder_id = get_folder_id(gg_folder_id, sub_folder)
+                # Create the folder if it doesn't exists
+                if not gg_sub_folder_id:
+                    print('creating folder ' + sub_folder)
+                    gg_sub_folder_id = create_folder(sub_folder, gg_folder_id)
+                recursive_walk_and_upload(sub_folder_path, gg_sub_folder_id,
+                                          processed_path, ignore_list)
+        if files:
+            for file in files:
+                filePath = os.path.join(root, file)
+                upload_file(filePath, gg_folder_id, ignore_list)
+        processed_path.append(root)
+
+
+def upload_folder_to_drive(local_folder, gg_folder_id,
+                           content_only=True,
+                           ignore_file=None):
+    """
+       Upload folder to Google Drive folder
+       bool content_only: if true, we only upload files and folder inside local_folder
+       else create a folder with the same name of the local folder and upload all files and folders
+       in the local folder to it
+   """
+
+    ignore_list = None
+    if ignore_file:
+        if filesys.is_file(ignore_file):
+            ignore_list = textfile.read_line_by_line(ignore_file)
+            ignore_list = [os.path.normpath(path) for path in ignore_list]
+
+    if content_only is False:
+        folder_name = filesys.get_dir_name(local_folder)
+        gg_folder_id_to_upload = create_folder(folder_name, gg_folder_id)
+    else:
+        gg_folder_id_to_upload = gg_folder_id
+
+    processed_path = []
+    local_folder = os.path.normpath(local_folder)
+    recursive_walk_and_upload(local_folder, gg_folder_id_to_upload,
+                              processed_path, ignore_list)

halib/gdrive_mkdir.py

@@ -0,0 +1,41 @@
+from argparse import ArgumentParser
+from datetime import datetime
+
+from halib.online import gdrive
+from halib.filetype import textfile
+
+
+def parse_args():
+    parser = ArgumentParser(
+        description="Upload local folder to Google Drive")
+    parser.add_argument('-a', '--authFile', type=str,
+                        help='authenticate file to Google Drive', default='settings.yaml')
+    parser.add_argument('-g', '--GDriveParentFolder', type=str,
+                        help='Destination parent folder ID in Google Drive')
+    parser.add_argument('-n', '--folderName', type=str,
+                        help='name of the folder which is about to be created',
+                        default='untitled')
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    auth_file = args.authFile
+    gDrive_parent_folder_id = args.GDriveParentFolder
+    folder_name = args.folderName
+
+    if folder_name == 'untitled':
+        folder_name = datetime.today().strftime('%Y.%m.%d_%Hh%M')
+    else:
+        date_str = datetime.today().strftime('%Y.%m.%d_%Hh%M')
+        folder_name = f'{date_str}_{folder_name}'
+
+    print(f'[GDrive] creating {folder_name} in GDrive folder {gDrive_parent_folder_id}')
+
+    gdrive.get_gg_drive(auth_file)
+    folder_id = gdrive.create_folder(folder_name, gDrive_parent_folder_id)
+    textfile.write([folder_id], './GDriveFolder.txt')
+
+
+if __name__ == "__main__":
+    main()

halib/gdrive_test.py

@@ -0,0 +1,37 @@
+from argparse import ArgumentParser
+
+from halib.online import gdrive
+
+
+def parse_args():
+    parser = ArgumentParser(
+        description="Upload local folder to Google Drive")
+    parser.add_argument('-a', '--authFile', type=str,
+                        help='authenticate file to Google Drive', default='settings.yaml')
+    parser.add_argument('-s', '--source', type=str,
+                        help='Folder to upload')
+    parser.add_argument('-d', '--destination', type=str,
+                        help='Destination folder ID in Google Drive')
+    parser.add_argument('-c', '--contentOnly', type=str,
+                        help='Parent Folder in Google Drive', default="True")
+    parser.add_argument('-i', '--ignoreFile', type=str,
+                        help='file containing files/folders to ignore', default=None)
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    auth_file = args.authFile
+    local_folder = args.source
+    gg_folder_id = args.destination
+    content_only = (args.contentOnly.lower() == "true")
+    ignore_file = args.ignoreFile
+    gdrive.get_gg_drive(auth_file)
+    gdrive.upload_folder_to_drive(local_folder, gg_folder_id,
+                                  content_only=content_only,
+                                  ignore_file=ignore_file)
+
+
+if __name__ == "__main__":
+    main()

halib/jsonfile.py

@@ -0,0 +1,22 @@
+import json
+
+
+def read(file):
+    with open(file) as f:
+        data = json.load(f)
+        return data
+
+
+def write(data_dict, outfile):
+    with open(outfile, 'w') as json_file:
+        json.dump(data_dict, json_file)
+
+
+def beautify(json_str):
+    formatted_json = json_str
+    try:
+        parsed = json.loads(json_str)
+        formatted_json = json.dumps(parsed, indent=4, sort_keys=True)
+    except Exception as e:
+        pass
+    return formatted_json

halib/listop.py

@@ -0,0 +1,13 @@
+def subtract(list_a, list_b):
+    return [item for item in list_a if item not in list_b]
+
+
+def union(list_a, list_b, no_duplicate=False):
+    if no_duplicate:
+        return list(set(list_a) | set(list_b))
+    else:
+        return list_a + list_b
+
+
+def intersection(list_a, list_b):
+    return list(set(list_a) & set(list_b))

halib/online/gdrive.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+
+"""
+    Upload folder to Google Drive
+"""
+import ast
+import os
+import requests
+import json
+
+import googleapiclient.errors
+
+# Import Google libraries
+from pydrive.auth import GoogleAuth
+from pydrive.drive import GoogleDrive
+from pydrive.files import GoogleDriveFileList
+
+from ..system import filesys
+from ..filetype import textfile
+
+# Import general libraries
+
+ggDrive = None
+ggAuth = None
+
+
+def get_gg_drive(settings_file="settings.yaml"):
+    """
+    Authenticate to Google API
+    """
+    global ggDrive
+    global ggAuth
+    if ggDrive is None:
+        ggAuth = GoogleAuth(settings_file=settings_file)
+        ggDrive = GoogleDrive(ggAuth)
+    return ggDrive
+
+
+def get_folder_id(gg_parent_folder_id, folder_name):
+    """
+    Check if destination folder exists and return it's ID
+    """
+
+    # Auto-iterate through all files in the parent folder.
+    drive = get_gg_drive()
+    file_list = GoogleDriveFileList()
+    try:
+        file_list = drive.ListFile(
+            {"q": "'{0}' in parents and trashed=false".format(gg_parent_folder_id)}
+        ).GetList()
+    # Exit if the parent folder doesn't exist
+    except googleapiclient.errors.HttpError as err:
+        # Parse error message
+        message = ast.literal_eval(err.content)["error"]["message"]
+        if message == "File not found: ":
+            print(message + folder_name)
+            exit(1)
+        # Exit with stacktrace in case of other error
+        else:
+            raise
+
+    # Find the the destination folder in the parent folder's files
+    for file1 in file_list:
+        if file1["title"] == folder_name:
+            print("title: %s, id: %s" % (file1["title"], file1["id"]))
+            return file1["id"]
+
+
+def create_folder(folder_name, gg_parent_folder_id):
+    """
+    Create folder on Google Drive
+    """
+
+    folder_metadata = {
+        "title": folder_name,
+        # Define the file type as folder
+        "mimeType": "application/vnd.google-apps.folder",
+        # ID of the parent folder
+        "parents": [{"kind": "drive#fileLink", "id": gg_parent_folder_id}],
+    }
+    drive = get_gg_drive()
+    folder = drive.CreateFile(folder_metadata)
+    folder.Upload()
+
+    # Return folder information
+    # print('title: %s, id: %s' % (folder['title'], folder['id']))
+    return folder["id"]
+
+
+def is_in_ignore_list(local_path, ignore_list=None):
+    in_ignore_list = False
+    if ignore_list:
+        for path in ignore_list:
+            if path in local_path:
+                in_ignore_list = True
+                break
+    return in_ignore_list
+
+
+def upload_file(local_file_path, gg_folder_id, ignore_list=None):
+    """
+    Upload local file to Google Drive folder
+    """
+    drive = get_gg_drive()
+    if not is_in_ignore_list(local_file_path, ignore_list):
+        link = ""
+
+        # print('uploading ' + local_file_path)
+        try:
+            # Upload file to folder.
+            title = filesys.get_file_name(local_file_path, split_file_ext=False)
+
+            # delete file if exist on gg folder
+            query = f"'{gg_folder_id}'  in parents and trashed=false"
+            file_list = drive.ListFile({"q": f"{query}"}).GetList()
+            for file in file_list:
+                if file["title"] == title:
+                    # print(f'[DELETE] {title} on Google Drive')
+                    file.Delete()
+                    break
+            # print('uploading ' + local_file_path)
+            f = drive.CreateFile(
+                {
+                    "title": f"{title}",
+                    "parents": [{"kind": "drive#fileLink", "id": gg_folder_id}],
+                }
+            )
+            f.SetContentFile(local_file_path)
+            f.Upload(param={"supportsAllDrives": True})
+            access_token = (
+                ggAuth.credentials.access_token
+            )  # gauth is from drive = GoogleDrive(gauth) Please modify this for your actual script.
+            # print(f'access_token {access_token}')
+            file_id = f["id"]
+            url = (
+                "https://www.googleapis.com/drive/v3/files/"
+                + file_id
+                + "/permissions?supportsAllDrives=true"
+            )
+            headers = {
+                "Authorization": "Bearer " + access_token,
+                "Content-Type": "application/json",
+            }
+            payload = {"type": "anyone", "value": "anyone", "role": "reader"}
+            res = requests.post(url, data=json.dumps(payload), headers=headers)
+            # SHARABLE LINK
+            link = f["alternateLink"]
+            return link
+        except Exception as e:
+            print("error uploading " + local_file_path)
+            print(e)
+            return link
+    # Skip the file if it's empty
+    else:
+        print("file {0} is empty or in ignore list".format(local_file_path))
+        return ""  # return empty string
+
+
+def recursive_walk_and_upload(
+    local_folder_path, gg_folder_id, processed_path, ignore_list=None
+):
+    for root, sub_folders, files in os.walk(local_folder_path):
+        # already processed folder
+        if root in processed_path:
+            print(f"[SKIP] already processed folder {root}")
+            return
+        print(f"\n\n[RECURSIVE] {local_folder_path}, {gg_folder_id}")
+        print(f"[FF] {root} {sub_folders} {files}")
+        if sub_folders:
+            for sub_folder in sub_folders:
+                sub_folder_path = os.path.join(root, sub_folder)
+                print(f"process {sub_folder_path}")
+                if is_in_ignore_list(sub_folder_path, ignore_list):
+                    continue
+                # Get destination folder ID
+                gg_sub_folder_id = get_folder_id(gg_folder_id, sub_folder)
+                # Create the folder if it doesn't exists
+                if not gg_sub_folder_id:
+                    print("creating folder " + sub_folder)
+                    gg_sub_folder_id = create_folder(sub_folder, gg_folder_id)
+                recursive_walk_and_upload(
+                    sub_folder_path, gg_sub_folder_id, processed_path, ignore_list
+                )
+        if files:
+            for file in files:
+                filePath = os.path.join(root, file)
+                upload_file(filePath, gg_folder_id, ignore_list)
+        processed_path.append(root)
+
+
+def upload_folder_to_drive(
+    local_folder, gg_folder_id, content_only=True, ignore_file=None
+):
+    """
+    Upload folder to Google Drive folder
+    bool content_only: if true, we only upload files and folder inside local_folder
+    else create a folder with the same name of the local folder and upload all files and folders
+    in the local folder to it
+    """
+
+    ignore_list = None
+    if ignore_file:
+        if filesys.is_file(ignore_file):
+            ignore_list = textfile.read_line_by_line(ignore_file)
+            ignore_list = [os.path.normpath(path) for path in ignore_list]
+
+    if content_only is False:
+        folder_name = filesys.get_dir_name(local_folder)
+        gg_folder_id_to_upload = create_folder(folder_name, gg_folder_id)
+    else:
+        gg_folder_id_to_upload = gg_folder_id
+
+    processed_path = []
+    local_folder = os.path.normpath(local_folder)
+    recursive_walk_and_upload(
+        local_folder, gg_folder_id_to_upload, processed_path, ignore_list
+    )
+
+
+def main():
+    settingf = r"halib\online\settings.yaml"
+    folder_id = "1RXew5llcebEXclbEAKQ2IWDtTWmW0d_z"
+    get_gg_drive(settingf)
+    sharelink = upload_file(r"D:\Dev\github_proj\halib\LICENSE.txt", folder_id)
+    print(sharelink)
+
+
+if __name__ == "__main__":
+    main()

halib/online/gdrive_mkdir.py

@@ -0,0 +1,53 @@
+from argparse import ArgumentParser
+from datetime import datetime
+
+import gdrive
+from ..filetype import textfile
+
+
+def parse_args():
+    parser = ArgumentParser(description="Upload local folder to Google Drive")
+    parser.add_argument(
+        "-a",
+        "--authFile",
+        type=str,
+        help="authenticate file to Google Drive",
+        default="settings.yaml",
+    )
+    parser.add_argument(
+        "-g",
+        "--GDriveParentFolder",
+        type=str,
+        help="Destination parent folder ID in Google Drive",
+    )
+    parser.add_argument(
+        "-n",
+        "--folderName",
+        type=str,
+        help="name of the folder which is about to be created",
+        default="untitled",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    auth_file = args.authFile
+    gDrive_parent_folder_id = args.GDriveParentFolder
+    folder_name = args.folderName
+
+    if folder_name == "untitled":
+        folder_name = datetime.today().strftime("%Y.%m.%d_%Hh%M")
+    else:
+        date_str = datetime.today().strftime("%Y.%m.%d_%Hh%M")
+        folder_name = f"{date_str}_{folder_name}"
+
+    print(f"[GDrive] creating {folder_name} in GDrive folder {gDrive_parent_folder_id}")
+
+    gdrive.get_gg_drive(auth_file)
+    folder_id = gdrive.create_folder(folder_name, gDrive_parent_folder_id)
+    textfile.write([folder_id], "./GDriveFolder.txt")
+
+
+if __name__ == "__main__":
+    main()