osbot-utils 1.7.7__py3-none-any.whl

osbot_utils/utils/Http.py

@@ -0,0 +1,136 @@
+import json
+import socket
+import ssl
+from time import sleep
+from   urllib.request import Request, urlopen
+
+from osbot_utils.utils.Files import save_bytes_as_file, file_size, file_bytes, file_open_bytes, file_create
+from osbot_utils.utils.Python_Logger import Python_Logger
+
+URL_CHECK_HOST_ONLINE = 'https://www.google.com'
+
+def current_host_offline(url_to_use=URL_CHECK_HOST_ONLINE):
+    return current_host_online(url_to_use=url_to_use) is False
+
+def current_host_online(url_to_use=URL_CHECK_HOST_ONLINE):
+    try:
+        http_request(url_to_use, method='HEAD')
+        return True
+    except:
+        return False
+
+def dns_ip_address(host):
+    return socket.gethostbyname(host)
+
+def is_port_open(host, port, timeout=0.5):
+    return port_is_open(host=host, port=port, timeout=timeout)
+
+def port_is_open(port : int , host='0.0.0.0', timeout=1.0):
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(timeout)
+        result = sock.connect_ex((host, port))
+        return result == 0
+    except:
+        return False
+
+
+def http_request(url, data=None, headers=None, method='GET', encoding ='utf-8', return_response_object=False):
+    ssl_request = url.startswith('https://')
+    headers = headers or {}
+    if data:
+        print()
+        if type(data) is not str:                                   # if the data object is not a string
+            if headers.get('Content-Type') == "application/json":   # and a json payload is expected
+                data = json.dumps(data)                             # convert it to json
+        if type(data) is str:                                       # only convert to bytes if current data is a string
+            data = data.encode()
+    request  = Request(url, data=data, headers=headers)
+    request.get_method = lambda: method
+
+    if ssl_request:
+        gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+        response = urlopen(request, context=gcontext)
+    else:
+        response = urlopen(request)
+
+    if return_response_object:
+        return response
+    else:
+        result = response.read()
+        if encoding:
+            return result.decode(encoding)
+        return result
+
+def port_is_not_open(port, host='0.0.0.0', timeout=1.0):
+    return port_is_open(port, host,timeout) is False
+
+def wait_for_http(url, max_attempts=20, wait_for=0.1):
+    for i in range(max_attempts):
+        try:
+            if GET(url):
+                return True
+        except:
+            pass
+        sleep(wait_for)
+    return False
+
+def wait_for_ssh(host, max_attempts=120, wait_for=0.5):
+    return wait_for_port(host=host, port=22, max_attempts=max_attempts, wait_for=wait_for)
+
+def wait_for_port(host, port, max_attempts=20, wait_for=0.1):
+    for i in range(max_attempts):
+        if is_port_open(host=host,port=port,timeout=wait_for):
+            return True
+        sleep(wait_for)
+    return False
+
+def wait_for_port_closed(host, port, max_attempts=20, wait_for=0.1):
+    for i in range(max_attempts):
+        if is_port_open(host=host,port=port,timeout=wait_for) is False:
+            return True
+        sleep(wait_for)
+    return False
+
+def DELETE(url, data=None, headers=None):
+    return http_request(url, data, headers, 'DELETE')
+
+def DELETE_json(*args, **kwargs):
+    return json.loads(DELETE(*args, **kwargs))
+
+def GET(url,headers = None, encoding='utf-8'):
+    return http_request(url, headers=headers, method='GET', encoding=encoding)
+
+def GET_to_file(url,path=None, headers = None, extension=None):
+    contents = GET(url, headers)
+    return file_create(path=path, contents=contents,extension=extension)
+
+def GET_bytes(url, headers=None):
+    return GET(url, headers=headers, encoding=None)
+
+def GET_bytes_to_file(url,path=None, headers = None):
+    file_bytes = GET_bytes(url, headers)
+    return save_bytes_as_file(file_bytes, path)
+
+def GET_json(*args, **kwargs):
+    return json.loads(GET(*args, **kwargs))
+
+def OPTIONS(url,headers = None):
+    response = http_request(url, headers=headers, method='OPTIONS', return_response_object=True)
+    response_headers  = {}
+    for response_header in response.getheaders():
+        (name,value) = response_header
+        response_headers[name] = value
+    return response_headers
+
+def POST(url, data='', headers=None):
+    return http_request(url, data, headers, 'POST')
+
+def POST_json(*args, **kwargs):
+    return json.loads(POST(*args, **kwargs))
+
+def PUT(url, data='', headers=None):
+    return http_request(url, data, headers, 'PUT')
+
+def PUT_json(*args, **kwargs):
+    return json.loads(PUT(*args, **kwargs))

osbot_utils/utils/Int.py

@@ -0,0 +1,6 @@
+@staticmethod
+def int_is_even(number):
+    return number % 2 == 0
+
+def int_is_odd(number):
+    return int_is_even(number) is False

osbot_utils/utils/Json.py

@@ -0,0 +1,171 @@
+import json
+import gzip
+import logging
+import os
+
+from osbot_utils.utils.Misc import str_lines, str_md5, str_sha256
+from osbot_utils.utils.Status import log_exception
+
+logger_json = logging.getLogger()   # todo: start using this API for capturing error messages from methods bellow
+
+from osbot_utils.utils.Files import file_create_gz, file_create, load_file_gz, file_contents, file_lines, file_lines_gz
+
+
+def json_dumps(python_object, indent=4, pretty=True, sort_keys=False, default=str, raise_exception=False):
+    if python_object:
+        try:
+            if pretty:
+                return json.dumps(python_object, indent=indent, sort_keys=sort_keys, default=default)
+            return json.dumps(python_object, default=default)
+        except Exception as error:
+            error_message = f'Error in load_json: {error}'
+            log_exception(message=error_message, error=error)
+            if raise_exception:
+                raise error
+
+
+def json_dumps_to_bytes(*args, **kwargs):
+    return json_dumps(*args, **kwargs).encode()
+
+def json_lines_file_load(target_path):
+    raw_json = '['                                          # start the json array
+    lines    = file_lines(target_path)                      # get all lines from the file provided in target_path
+    raw_json += ','.join(lines)                             # add lines to raw_json split by json array separator
+    raw_json += ']'                                         # close the json array
+    return json_parse(raw_json)                             # convert json data into a python object
+
+def json_lines_file_load_gz(target_path):
+    raw_json = '['                                          # start the json array
+    lines    = file_lines_gz(target_path)                      # get all lines from the file provided in target_path
+    raw_json += ','.join(lines)                             # add lines to raw_json split by json array separator
+    raw_json += ']'                                         # close the json array
+    return json_parse(raw_json)                             # convert json data into a python object
+
+
+def json_sha_256(target):
+    return str_sha256(json_dumps(target))
+
+
+
+class Json:
+
+    @staticmethod
+    def load_file(path):
+        """
+        Loads json data from file
+        Note: will not throw errors and will return {} as default
+        errors are logged to Json.log
+        """
+        json_data = file_contents(path)
+        return json_loads(json_data)
+
+    @staticmethod
+    def load_file_and_delete(path):
+        data = json_load_file(path)
+        if data:
+            os.remove(path)
+        return data
+
+    @staticmethod
+    def load_file_gz(path):
+        data = load_file_gz(path)
+        return json_loads(data)
+
+    @staticmethod
+    def load_file_gz_and_delete(path):
+        data = json_load_file_gz(path)
+        if data:
+            os.remove(path)
+        return data
+
+    @staticmethod
+    def loads(json_data, raise_exception=False):
+        """
+        Loads json data from string
+        Note: will not throw errors and will return {} as default
+        errors are logged to Json.log
+        """
+        if json_data:
+            try:
+                return json.loads(json_data)
+            except Exception as error:
+                log_exception(message='Error in load_json', error=error)
+                if raise_exception:
+                    raise error
+
+        return {}
+
+    @staticmethod
+    def loads_json_lines(json_lines):
+        json_data = '[' + ','.join(str_lines(json_lines.strip())) + ']'
+        return json_loads(json_data)
+
+    @staticmethod
+    def md5(data):
+        return str_md5(json_dump(data))
+
+    @staticmethod
+    def round_trip(data):
+        return json_loads(json_dumps(data))
+
+    @staticmethod
+    def save_file(python_object, path=None, pretty=False, sort_keys=False):
+        json_data = json_dumps(python_object=python_object, indent=2, pretty=pretty, sort_keys=sort_keys)
+        return file_create(path=path, contents=json_data)
+
+    @staticmethod
+    def save_file_pretty(python_object, path=None):
+        return json_save_file(python_object=python_object, path=path, pretty=True)
+
+    @staticmethod
+    def save_file_gz(python_object, path=None, pretty=False):
+        json_data = json_dumps(python_object,indent=2, pretty=pretty)
+        return file_create_gz(path=path, contents=json_data)
+
+    @staticmethod
+    def save_file_pretty_gz(python_object, path=None):
+        return json_save_file_gz(python_object=python_object, path=path, pretty=True)
+
+
+    @staticmethod
+    def json_save_tmp_file(python_object, pretty=True):
+        return Json.save_file(python_object=python_object, pretty=pretty, path=None)
+
+file_create_json             = Json.save_file_pretty
+file_contents_json           = Json.load_file
+
+json_dump                    = json_dumps
+json_format                  = json_dumps
+json_file_create             = Json.save_file
+json_file_create_gz          = Json.save_file_gz
+json_file_contents           = Json.load_file
+json_file_contents_gz        = Json.load_file_gz
+json_file_load               = Json.load_file
+json_file_safe               = Json.save_file
+json_from_file               = Json.load_file
+json_load_file               = Json.load_file
+json_load_file_and_delete    = Json.load_file_and_delete
+json_load_file_gz            = Json.load_file_gz
+json_load_file_gz_and_delete = Json.load_file_gz_and_delete
+json_from_string             = Json.loads
+json_load                    = Json.loads
+json_loads                   = Json.loads
+json_md5                     = Json.md5
+json_lines_loads             = Json.loads_json_lines
+json_parse                   = Json.loads
+json_lines_parse             = Json.loads_json_lines
+json_to_str                  = json_dumps
+json_round_trip              = Json.round_trip
+json_save                    = Json.save_file
+json_save_file               = Json.save_file
+json_save_file_pretty        = Json.save_file_pretty
+json_save_file_gz            = Json.save_file_gz
+json_save_file_pretty_gz     = Json.save_file_pretty_gz
+json_save_tmp_file           = Json.json_save_tmp_file
+str_to_json                  = Json.loads
+
+load_file_json               = json_load_file
+load_file_json_gz            = json_load_file_gz
+
+to_json_str                  = json_dumps
+from_json_str                = json_loads

osbot_utils/utils/Json_Cache.py

@@ -0,0 +1,59 @@
+from osbot_utils.utils.Files import path_combine, folder_create, file_delete, file_exists
+from osbot_utils.utils.Json import json_save_file_gz, json_save_file, json_load_file_gz, json_load_file
+from osbot_utils.utils.Str import str_safe
+
+PATH_TEMP_FOLDER = '/tmp/json_cache'
+
+class Json_Cache:
+
+    def __init__(self, cache_type=None, cache_keys=None):
+        self.cache_type      = cache_type or '__cache'
+        self.cache_keys      = cache_keys or [self.cache_type]
+        self.path_tmp_folder = PATH_TEMP_FOLDER
+        self.save_as_gz      = True
+        if type(self.cache_keys) is not list and type(self.cache_keys) is not tuple:
+            self.cache_keys = [self.cache_keys]
+        #self.data = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def data(self):
+        file_path = self.path_cache_file()
+        if self.exists():
+            if self.save_as_gz:
+                return json_load_file_gz(file_path)
+            else:
+                return json_load_file(file_path)
+        return None
+
+    def delete(self):
+        return file_delete(self.path_cache_file())
+
+    def exists(self):
+        return file_exists(self.path_cache_file())
+
+    def save(self, data):
+        file_path = self.path_cache_file()
+        if self.save_as_gz:
+            return json_save_file_gz(data, file_path)
+        else:
+            return json_save_file(data, file_path)
+
+    def path_cache_folder(self):
+        path_cache = path_combine(self.path_tmp_folder, self.cache_type)
+        folder_create(path_cache)
+        return path_cache
+
+    def path_cache_file(self):
+        tmp_file_name = ""#str_safe(str(self.cache_type))
+        for cache_key in  self.cache_keys:
+            tmp_file_name += f"--{str_safe(str(cache_key))}"
+        tmp_file_name += ".json"
+        if self.save_as_gz:
+            tmp_file_name += ".gz"
+        file_path = path_combine(self.path_cache_folder(), tmp_file_name)
+        return file_path

osbot_utils/utils/Lists.py

@@ -0,0 +1,198 @@
+import sys
+from typing import List
+
+from osbot_utils.utils.Misc import list_set
+
+from osbot_utils.utils.Str     import trim
+
+def len_list(target):
+    if type(target) is list:
+        return len(list(target))
+    return 0
+
+def list_add(array : list, value):
+    if value is not None:
+        array.append(value)
+    return value
+
+def list_chunks(items:list, split: int):
+    if items and split and split > 0:
+        for i in range(0, len(items), split):
+            yield items[i:i + split]
+
+def list_contains_list(array : list, values):
+    if array is not None:
+        if type(values) is list:
+            for item in values:
+                if (item in array) is False:
+                    return False
+            return True
+    return False
+
+def list_delete(target, item):
+    if item in target:
+        target.remove(item)
+    return target
+
+def list_empty(list):
+    return not list_not_empty(list)
+
+def list_filter(target_list, filter_function):
+    return list(filter(filter_function, target_list))
+
+def list_filter_starts_with(target_list, prefix):
+    return list_filter(target_list, lambda x: x.startswith(prefix))
+
+def list_filter_contains(target_list, value):
+    return list_filter(target_list, lambda x: x.find(value) > -1)
+
+def list_find(array:list, item):
+    if item in array:
+        return array.index(item)
+    return -1
+
+def list_first(list, strip=False):
+    if list_not_empty(list):
+        value = list[0]
+        if strip:
+            value = value.strip()
+        return value
+
+def list_in_list(source : list, target: list):
+    return set(source).issubset(set(target))
+
+def list_get(array, position=None, default=None):
+    if type(array) is list:
+        if type(position) is int and position >=0 :
+            if  len(array) > position:
+                return array[position]
+    return default
+
+def list_get_field(values, field):
+    if type(values) is list:
+        return [item.get(field) for item in values]
+    return []
+
+def list_group_by(values, group_by):
+    results = {}
+    if type(values) is list:
+        for item in values:
+            value = str(item.get(group_by))
+            if results.get(value) is None: results[value] = []
+            results[value].append(item)
+    return results
+
+def list_index_by(values, index_by):
+    from osbot_utils.fluent.Fluent_Dict import Fluent_Dict
+    results = {}
+    if values and index_by:
+        for item in values:
+            results[item.get(index_by)] = item
+    return Fluent_Dict(results)
+
+def list_lower(input_list):
+    return [item.lower() for item in input_list]
+
+def list_not_empty(list):
+    if list and type(list).__name__ == 'list' and len(list) >0:
+        return True
+    return False
+
+def list_order_by(target: List[dict], key: str, reverse: bool=False) -> List[dict]:
+    if target and key:
+        return sorted(target, key=lambda x: x[key], reverse=reverse)
+    return []
+
+def list_pop(array:list, position=None, default=None):
+    if array:
+        if len(array) >0:
+            if type(position) is int:
+                if len(array) > position:
+                    return array.pop(position)
+            else:
+                return array.pop()
+    return default
+
+def list_pop_and_trim(array, position=None):
+    value = array_pop(array,position)
+    if type(value) is str:
+        return trim(value)
+    return value
+
+def list_remove(array, item):
+    if type(array) is list:
+        if type(item) is list:
+            result = []
+            for element in array:
+                if element not in item:
+                    result.append(element)
+            return result
+
+        return [element for element in array if element != item]
+    return array
+
+def list_remove_list(source: list, target: list):
+    if type(source) is list and type(target) is list:
+        for item in target:
+            if item in source:
+                source.remove(item)
+    return source
+
+def list_remove_empty(array):
+    if type(array) is list:
+        return [element for element in array if element]
+    return array
+
+def list_set_dict(target):
+    if hasattr(target, '__dict__'):
+        return sorted(list(set(target.__dict__)))
+    return []
+
+def list_sorted(target_list, key, descending=False):
+    return list(sorted(target_list, key= lambda x:x.get(key,None) ,reverse=descending))
+
+def list_stats(target):
+    stats = {}
+    if type(target) is list:
+        for item in target:
+            if stats.get(item) is None:
+                stats[item] = 0
+            stats[item] += 1
+    return stats
+
+def list_to_tuple(target: list):
+    if type(target) is list:
+        return tuple(target)
+
+def list_zip(*args):
+    if args:
+        return list(zip(*args))
+
+def sys_path_python(python_folder='lib/python'):
+    return list_contains(sys.path, python_folder)
+
+def tuple_to_list(target:tuple):
+    if type(target) is tuple:
+        return list(target)
+
+def tuple_replace_position(target:tuple, position,value):
+    tuple_as_list = tuple_to_list(target)
+    if len(tuple_as_list) > position:
+        tuple_as_list[position] = value
+    list_as_tuple = list_to_tuple(tuple_as_list)
+    return list_as_tuple
+
+def unique(target):
+    return list_set(target)
+
+array_find             = list_find
+array_get              = list_get
+array_pop              = list_pop
+array_pop_and_trim     = list_pop_and_trim
+array_add              = list_add
+
+list_contains          = list_filter_contains
+list_del               = list_delete
+list_sort_by           = list_sorted
+
+chunks                 = list_chunks