hsds 0.9.1__py3-none-any.whl

admin/config/config.yml

@@ -0,0 +1,106 @@
+# HSDS configuration
+allow_noauth: true # enable unauthenticated requests
+auth_expiration: -1 # set an expiration for credential caching
+default_public: False # new domains are publically readable by default
+aws_access_key_id: xxx # Replace with access key for account or use aws_iam_role
+aws_secret_access_key: xxx # Replace with secret key for account
+aws_iam_role: hsds_role # For EC2 using IAM roles
+aws_region: us-east-1
+hsds_endpoint: http://hsds.hdf.test # used for hateos links in response
+aws_s3_gateway: null # use endpoint for the region HSDS is running in, e.g. 'https://s3.amazonaws.com' for us-east-1
+aws_s3_no_sign_request: false # do not use credentials for S3 requests, equivalent of --no-sign-request for AWS CLI
+aws_dynamodb_gateway: null # use for dynamodb endpint, e.g. 'https://dynamodb.us-east-1.amazonaws.com',
+aws_dynamodb_users_table: null # set to table name if dynamodb is used to store usernames and passwords
+azure_connection_string: null # use for connecting to Azure blob storage
+azure_resource_id: null # resource id for use with Azure Active Directory
+azure_storage_account: null # storage account to use on Azure
+azure_resource_group: null # Azure resource group the container (BUCKET_NAME) belongs to
+root_dir: null # base directory to use for Posix storage
+password_salt: null # salt value to generate password based on username.  Not recommended for public deployments
+bucket_name: hsdstest # set to use a default bucket, otherwise bucket param is needed for all requests
+head_port: 5100 # port to use for head node
+head_ram: 512m # memory for head container
+dn_port: 6101 # Start dn ports at 6101
+dn_ram: 3g # memory for DN container (per container)
+sn_port: 5101 # Start sn ports at 5101
+sn_ram: 3g # memory for SN container
+target_sn_count: 0 # desired number of SN containers
+target_dn_count: 0 # desire number of DN containers
+log_level: INFO    # log level.  One of ERROR, WARNING, INFO, DEBUG
+log_timestamps: false # emit timestamp with log messages
+log_prefix: null # Prefix text to append to log entries
+max_tcp_connections: 100 # max number of inflight tcp connections
+head_sleep_time: 10 # max sleep time between health checks for head node
+node_sleep_time: 10 # max sleep time between health checks for SN/DN nodes
+async_sleep_time: 1 # max sleep time between async task runs
+scan_sleep_time: 10  # max sleep time between scanning runs
+scan_wait_time: 10   # min time to wait after a domain update before starting a scan
+max_scan_duration: 180 # max time to wait for a scan to complete before raising error
+gc_sleep_time: 10   # max time between runs to delete unused objects
+s3_sync_interval: 1 # time to wait between s3_sync checks (in sec)
+s3_age_time: 1 # time to wait since last update to write an object to S3
+s3_sync_task_timeout: 10 # time to cancel write task if no response
+store_read_timeout: 1 # time to cancel storage read request if no response
+store_read_sleep_interval: 0.1 # time to sleep between checking on read request
+max_pending_write_requests: 20 # maxium number of inflight write requests
+flush_sleep_interval: 1 # time to wait between checking on dirty objects
+flush_timeout: 10 # max time to wait on all I/O operations to complete for a flush
+min_chunk_size: 1m # 1 MB
+max_chunk_size: 4m # 4 MB
+max_request_size: 100m # 100 MB - should be no smaller than client_max_body_size in nginx tmpl (if using nginx)
+max_chunks_per_folder: 0 # max number of chunks per s3 folder. 0 for unlimiited
+max_task_count: 100 # maximum number of concurrent tasks per node before server will return 503 error
+max_tasks_per_node_per_request: 16 # maximum number of inflight tasks to each node per request
+aio_max_pool_connections: 64 # number of connections to keep in conection pool for aiobotocore requests
+client_pool_count: 10 # pool count for SessionClient
+metadata_mem_cache_size: 128m # 128 MB - metadata cache size per DN node
+metadata_mem_cache_expire: 3600 # expire cache items after one hour
+chunk_mem_cache_size: 128m # 128 MB - chunk cache size per DN node
+chunk_mem_cache_expire: 3600 # expire cache items after one hour
+timeout: 30 # http timeout - 30 sec
+password_file: /config/passwd.txt # filepath to a text file of username/passwords. set to '' for no-auth access
+groups_file: /config/groups.txt # filepath to text file defining user groups
+server_name: Highly Scalable Data Service (HSDS) # this gets returned in the about request
+greeting: Welcome to HSDS!
+about: HSDS is a webservice for HDF data
+top_level_domains: [] # list of possible top-level domains, example: ["/home", "/shared"], if empty all top-level folders in default bucket will be returned
+cors_domain: "*" # domains allowed for CORS
+admin_user: admin # user with admin privileges
+admin_group: null # enable admin privileges for any user in this group
+openid_provider: azure # OpenID authentication provider
+openid_url: null # OpenID connect endpoint if provider is not azure or google
+openid_audience: null # OpenID audience. This is synonymous with azure_resource_id for azure
+openid_claims: unique_name,appid,roles # Comma seperated list of claims to resolve to usernames.
+chaos_die: 0 # if > 0, have nodes randomly die after n seconds (for testing)
+standalone_app: false # True when run as a single application
+blosc_nthreads: 2 # number of threads to use for blosc compression.  Set to 0 to have blosc auto-determine thread count
+http_compression: false # Use HTTP compression
+http_max_url_length: 512 # Limit http request url + params to be less than this
+http_streaming: true  # enable HTTP streaming 
+k8s_dn_label_selector: app=hsds # Selector for getting data node pods from a k8s deployment (https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors)
+k8s_namespace: null # Specifies if a the client should be limited to a specific namespace. Useful for some RBAC configurations.
+restart_policy: on-failure # Docker restart policy
+# the following two values with give backoff times of approx: 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8
+dn_max_retries: 7 # number of time to retry DN requests
+dn_retry_backoff_exp: 0.1 # backoff factor for retries
+xss_protection: "1; mode=block"  # Include in response headers if set
+allow_any_bucket_read: true  # enable reads to buckets other than default bucket
+allow_any_bucket_write: true # enable writes to buckets other than default bucket
+bit_shuffle_default_blocksize: 2048 # default blocksize for bitshuffle filter
+max_rangeget_gap: 1024 # max gap in byte for intelligent range get requests
+# DEPRECATED - the remaining config values are not used in currently but kept for backward compatibility with older container images
+aws_lambda_chunkread_function: null # name of aws lambda function for chunk reading
+aws_lambda_threshold: 4 # number of chunks per node per request to reach before using lambda
+aws_lambda_max_invoke: 1000 # max number of lambda functions to invoke simultaneously
+aws_lambda_gateway: null # use lambda endpoint for region HSDS is running in
+k8s_app_label: null # The app label for k8s deployments (use k8s_dn_label_selector instead)
+write_zero_chunks: False # write chunk to storage even when it's all zeros (or in general equal to the fill value)
+max_chunks_per_request: 1000 # maximum number of chunks to be serviced by one request
+rangeget_port: 6900 # singleton proxy at port 6900
+rangeget_ram: 2g # memory for RANGEGET container
+data_cache_size: 128m # cache for rangegets
+data_cache_max_req_size: 128k # max size for rangeget fetches
+data_cache_expire_time: 3600 # expire cache items after one hour
+data_cache_page_size: 4m # page size for range get cache, set to zero to disable proxy
+data_cache_max_concurrent_read: 16 # maximum number of inflight storage read requests
+domain_req_max_objects_limit: 500 # maximum number of objects to return in GET domain request with use_cache

hsds/app.py

@@ -0,0 +1,385 @@
+##############################################################################
+# Copyright by The HDF Group.                                                #
+# All rights reserved.                                                       #
+#                                                                            #
+# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and      #
+# Utilities.  The full HSDS copyright notice, including                      #
+# terms governing use, modification, and redistribution, is contained in     #
+# the file COPYING, which can be found at the root of the source code        #
+# distribution tree.  If you do not have access to this file, you may        #
+# request a copy from help@hdfgroup.org.                                     #
+##############################################################################
+import argparse
+import os
+import json
+import sys
+import logging
+import time
+
+from .hsds_app import HsdsApp
+from . import config
+
+_HELP_USAGE = "Starts HSDS, a REST-based service for HDF5 data."
+
+_HELP_EPILOG = """Examples:
+
+- with a POSIX-based storage using a directory: ./hsdata for storage:
+
+  hsds --root_dir ~/hsdata
+
+- with POSIX-based storage and config settings and password file:
+
+  hsds --root_dir ~/hsdata --password_file ./admin/config/passwd.txt \
+  --config_dir ./admin/config
+
+- with minio data storage:
+
+  hsds --s3-gateway http://localhost:6007 --access-key-id demo:demo
+      --secret-access-key DEMO_PASS --password_file ./admin/config/passwd.txt
+
+- with AWS S3 storage and a bucket in the us-west-2 region:
+
+  hsds --s3-gateway http://s3.us-west-2.amazonaws.com --access-key-id ${AWS_ACCESS_KEY_ID} \
+      --secret-access-key ${AWS_SECRET_ACCESS_KEY} --password_file ./admin/config/passwd.txt
+
+"""
+
+# maximum number of characters if socket directory is given
+# Exceeding this can cause errors - see: https://github.com/HDFGroup/hsds/issues/129
+# Underlying issue is reported here: https://bugs.python.org/issue32958
+MAX_SOCKET_DIR_PATH_LEN = 63
+
+
+class UserConfig:
+    """
+    User Config state
+    """
+
+    def __init__(self, config_file=None, **kwargs):
+        self._cfg = {}
+        if config_file:
+            self._config_file = config_file
+        elif os.path.isfile(".hscfg"):
+            self._config_file = ".hscfg"
+        else:
+            self._config_file = os.path.expanduser("~/.hscfg")
+        # process config file if found
+        if os.path.isfile(self._config_file):
+            line_number = 0
+            with open(self._config_file) as f:
+                for line in f:
+                    line_number += 1
+                    s = line.strip()
+                    if not s:
+                        continue
+                    if s[0] == "#":
+                        # comment line
+                        continue
+                    index = line.find("=")
+                    if index <= 0:
+                        print(
+                            "config file: {} line: {} is not valid".format(
+                                self._config_file, line_number
+                            )
+                        )
+                        continue
+                    k = line[:index].strip()
+                    nlen = index + 1
+                    v = line[nlen:].strip()
+                    if v and v.upper() != "NONE":
+                        self._cfg[k] = v
+        # override any config values with environment variable if found
+        for k in self._cfg.keys():
+            if k.upper() in os.environ:
+                self._cfg[k] = os.environ[k.upper()]
+
+        # finally update any values that are passed in to the constructor
+        for k in kwargs.keys():
+            self._cfg[k.upper()] = kwargs[k]
+
+    def __getitem__(self, name):
+        """Get a config item"""
+
+        # Load a variable from environment. It would have only been loaded in
+        # __init__ if it was also specified in the config file.
+        env_name = name.upper()
+        if name not in self._cfg and env_name in os.environ:
+            self._cfg[name] = os.environ[env_name]
+
+        return self._cfg[name]
+
+    def __setitem__(self, name, obj):
+        """set config item"""
+        self._cfg[name] = obj
+
+    def __delitem__(self, name):
+        """Delete option."""
+        del self._cfg[name]
+
+    def __len__(self):
+        return len(self._cfg)
+
+    def __iter__(self):
+        """Iterate over config names"""
+        keys = self._cfg.keys()
+        for key in keys:
+            yield key
+
+    def __contains__(self, name):
+        return name in self._cfg or name.upper() in os.environ
+
+    def __repr__(self):
+        return json.dumps(self._cfg)
+
+    def keys(self):
+        return self._cfg.keys()
+
+    def get(self, name, default=None):
+        if name in self:
+            return self[name]
+        else:
+            return default
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawTextHelpFormatter,
+        usage=_HELP_USAGE,
+        epilog=_HELP_EPILOG,
+    )
+
+    parser.add_argument(
+        "--root_dir",
+        type=str,
+        dest="root_dir",
+        help="Directory where to store the object store data",
+    )
+    parser.add_argument(
+        "--bucket_name",
+        type=str,
+        dest="bucket_name",
+        help='Name of the bucket to use (e.g., "hsds.test").',
+    )
+    parser.add_argument(
+        "--host", default="", type=str, dest="host", help="host name for url"
+    )
+    parser.add_argument(
+        "--hs_username",
+        type=str,
+        dest="hs_username",
+        help="username to be added to list of valid users",
+        default="",
+    )
+    parser.add_argument(
+        "--hs_password",
+        type=str,
+        dest="hs_password",
+        help="password for hs_username",
+        default="",
+    )
+    parser.add_argument(
+        "--password_file",
+        type=str,
+        dest="password_file",
+        help="location of hsds password file",
+        default="",
+    )
+
+    parser.add_argument(
+        "--logfile",
+        default="",
+        type=str,
+        dest="logfile",
+        help="filename for logout (default stdout).",
+    )
+    parser.add_argument(
+        "--loglevel",
+        default="",
+        type=str,
+        dest="loglevel",
+        help="log verbosity: DEBUG, WARNING, INFO, OR ERROR",
+    )
+    parser.add_argument(
+        "-p", "--port", default=0, type=int, dest="port", help="Service node port"
+    )
+    parser.add_argument(
+        "--count",
+        default=4,
+        type=int,
+        dest="dn_count",
+        help="Number of dn sub-processes to create.",
+    )
+    parser.add_argument(
+        "--socket_dir",
+        default="",
+        type=str,
+        dest="socket_dir",
+        help="directory for socket endpoint",
+    )
+    parser.add_argument(
+        "--config_dir",
+        default="",
+        type=str,
+        dest="config_dir",
+        help="directory for config data",
+    )
+
+    args = parser.parse_args()
+
+    kwargs = {}  # options to pass to hsdsapp
+
+    sys.stdout.reconfigure(encoding='utf-8')
+    sys.stderr.reconfigure(encoding='utf-8')
+
+    # setup logging
+    if args.loglevel:
+        log_level_cfg = args.loglevel
+        kwargs["log_level"] = args.loglevel
+    elif "LOG_LEVEL" in os.environ:
+        log_level_cfg = os.environ["LOG_LEVEL"]
+    else:
+        log_level_cfg = "INFO"
+    if log_level_cfg == "DEBUG":
+        log_level = logging.DEBUG
+    elif log_level_cfg == "INFO":
+        log_level = logging.INFO
+    elif log_level_cfg in ("WARN", "WARNING"):
+        log_level = logging.WARN
+    elif log_level_cfg == "ERROR":
+        log_level = logging.ERROR
+    else:
+        print(f"unsupported log_level: {log_level_cfg}, using INFO instead")
+        log_level = logging.INFO
+
+    print("set logging to::", log_level)
+    logging.basicConfig(level=log_level)
+
+    userConfig = UserConfig()
+
+    login_username = None
+    try:
+        login_username = os.getlogin()
+    except OSError:
+        pass  # ignore
+
+    username = None
+    password = None
+
+    # validate password_file arg if used
+    if args.password_file:
+        if not os.path.isfile(args.password_file):
+            sys.exit(f"password file: {args.password_file} not found")
+        kwargs["password_file"] = args.password_file
+
+        if args.hs_username:
+            sys.exit("username arg cannot be used if password_file is set")
+        if args.hs_password:
+            sys.exit("password arg cannot be used if password_file is set")
+    else:
+        if args.hs_username:
+            username = args.hs_username
+        elif "HS_USERNAME" in userConfig:
+            # can set username based on HS_USERNAME environment variable
+            username = userConfig["HS_USERNAME"]
+        else:
+            username = login_username
+
+        if args.hs_password:
+            password = args.hs_password
+        elif "HS_PASSWORD" in userConfig:
+            # can set password based on HS_PASSWORD environment variable
+            password = userConfig["HS_PASSWORD"]
+        else:
+            password = login_username
+
+    if username and password:
+        kwargs["username"] = username
+        kwargs["password"] = password
+
+    # use unix domain socket if a socket dir is set
+    if args.socket_dir:
+        socket_dir = os.path.abspath(args.socket_dir)
+        if not os.path.isdir(socket_dir):
+            raise FileNotFoundError(f"directory: {socket_dir} not found")
+        kwargs["socket_dir"] = socket_dir
+    else:
+        # USE TCP connect
+        if args.host:
+            kwargs["host"] = args.host
+        else:
+            kwargs["host"] = "localhost"
+        # sn_port only relevant for TCP connections
+        if args.port:
+            kwargs["sn_port"] = args.port
+        else:
+            kwargs["sn_port"] = 5101  # TBD - use config
+
+    if args.logfile:
+        logfile = os.path.abspath(args.logfile)
+    elif args.host:
+        logfile = os.path.abspath("hs.log")
+    else:
+        socket_dir = os.path.abspath(args.socket_dir)
+        logfile = os.path.join(socket_dir, "hs.log")
+    print("logfile:", logfile)
+    kwargs["logfile"] = logfile
+
+    if args.root_dir:
+        kwargs["root_dir"] = args.root_dir
+
+    config_dir = None
+    if args.config_dir:
+        if not os.path.isdir(args.config_dir):
+            print(f"config_dir: {args.config_dir} not found")
+        else:
+            config_dir = args.config_dir
+    if config_dir:
+        kwargs["config_dir"] = config_dir
+
+    if args.dn_count:
+        kwargs["dn_count"] = args.dn_count
+
+    if args.bucket_name:
+        bucket_name = args.bucket_name
+    else:
+        bucket_name = config.get("bucket_name")
+    if not bucket_name:
+        sys.exit("bucket_name not set")
+    if args.root_dir:
+        root_dir = args.root_dir
+    else:
+        root_dir = config.get("root_dir")
+    if not root_dir:
+        # check that AWS_S3_GATEWAY or AZURE_CONNECTION_STRING is set
+        if not config.get("aws_s3_gateway") and not config.get("azure_connection_string"):
+            sys.exit("root_dir not set (and no S3 or Azure connection info)")
+    else:
+        if not os.path.isdir(root_dir):
+            sys.exit(f"directory: {root_dir} not found")
+        bucket_path = os.path.join(root_dir, bucket_name)
+        if not os.path.isdir(bucket_path):
+            os.mkdir(bucket_path)
+
+    app = HsdsApp(**kwargs)
+    app.run()
+
+    waiting_on_ready = True
+
+    while True:
+        try:
+            time.sleep(1)
+            app.check_processes()
+        except KeyboardInterrupt:
+            print("got keyboard interrupt")
+            break
+        except Exception as e:
+            print(f"got exception: {e}")
+            break
+        if waiting_on_ready and app.ready:
+            waiting_on_ready = False
+            print("")
+            print("READY! use endpoint:", app.endpoint)
+            print("")
+
+    print("shutting down server")
+    app.stop()