idmtools-test 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools_test/inputs/python/Assets/MyExternalLibrary/functions.py

@@ -0,0 +1,15 @@
+import numpy as np
+
+
+def add(x, y):
+    """
+    Add x to y
+
+    Args:
+        x: x to add
+        y: y to add
+
+    Returns:
+
+    """
+    return np.add(x, y)

idmtools_test/inputs/python/Assets/MyLib/functions.py

@@ -0,0 +1,2 @@
+def add(a: int, b: int):
+    return a + b

idmtools_test/inputs/python/Assets/MyLib/temp.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python
+# flake8: noqa W291,E701
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD.  The full license text is available at:
+#  - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+Methods for serialized objects (or source code) stored in temporary files
+and file-like objects.
+"""
+# XXX: better instead to have functions write to any given file-like object ?
+# XXX: currently, all file-like objects are created by the function...
+
+__all__ = ['dump_source', 'dump', 'dumpIO_source', 'dumpIO', \
+           'load_source', 'load', 'loadIO_source', 'loadIO', \
+           'capture']
+
+import contextlib
+from ._dill import PY3
+
+
+@contextlib.contextmanager
+def capture(stream='stdout'):
+    """builds a context that temporarily replaces the given stream name
+
+    >>> with capture('stdout') as out:
+    ...   print "foo!"
+    ... 
+    >>> print out.getvalue()
+    foo!
+
+    """
+    import sys
+    if PY3:
+        from io import StringIO
+    else:
+        from StringIO import StringIO
+    orig = getattr(sys, stream)
+    setattr(sys, stream, StringIO())
+    try:
+        yield getattr(sys, stream)
+    finally:
+        setattr(sys, stream, orig)
+
+
+def b(x):  # deal with b'foo' versus 'foo'
+    import codecs
+    return codecs.latin_1_encode(x)[0]
+
+
+def load_source(file, **kwds):
+    """load an object that was stored with dill.temp.dump_source
+
+    file: filehandle
+    alias: string name of stored object
+    mode: mode to open the file, one of: {'r', 'rb'}
+
+    >>> f = lambda x: x**2
+    >>> pyfile = dill.temp.dump_source(f, alias='_f')
+    >>> _f = dill.temp.load_source(pyfile)
+    >>> _f(4)
+    16
+    """
+    alias = kwds.pop('alias', None)
+    mode = kwds.pop('mode', 'r')
+    fname = getattr(file, 'name', file)  # fname=file.name or fname=file (if str)
+    source = open(fname, mode=mode, **kwds).read()
+    if not alias:
+        tag = source.strip().splitlines()[-1].split()
+        if tag[0] != '#NAME:':
+            stub = source.splitlines()[0]
+            raise IOError("unknown name for code: %s" % stub)
+        alias = tag[-1]
+    local = {}
+    exec(source, local)
+    _ = eval("%s" % alias, local)
+    return _
+
+
+def dump_source(object, **kwds):
+    """write object source to a NamedTemporaryFile (instead of dill.dump)
+Loads with "import" or "dill.temp.load_source".  Returns the filehandle.
+
+    >>> f = lambda x: x**2
+    >>> pyfile = dill.temp.dump_source(f, alias='_f')
+    >>> _f = dill.temp.load_source(pyfile)
+    >>> _f(4)
+    16
+
+    >>> f = lambda x: x**2
+    >>> pyfile = dill.temp.dump_source(f, dir='.')
+    >>> modulename = os.path.basename(pyfile.name).split('.py')[0]
+    >>> exec('from %s import f as _f' % modulename)
+    >>> _f(4)
+    16
+
+Optional kwds:
+    If 'alias' is specified, the object will be renamed to the given string.
+
+    If 'prefix' is specified, the file name will begin with that prefix,
+    otherwise a default prefix is used.
+    
+    If 'dir' is specified, the file will be created in that directory,
+    otherwise a default directory is used.
+    
+    If 'text' is specified and true, the file is opened in text
+    mode.  Else (the default) the file is opened in binary mode.  On
+    some operating systems, this makes no difference.
+
+NOTE: Keep the return value for as long as you want your file to exist !
+    """  # XXX: write a "load_source"?
+    from .source import importable, getname
+    import tempfile
+    kwds.pop('suffix', '')  # this is *always* '.py'
+    alias = kwds.pop('alias', '')  # XXX: include an alias so a name is known
+    name = str(alias) or getname(object)
+    name = "\n#NAME: %s\n" % name
+    # XXX: assumes kwds['dir'] is writable and on $PYTHONPATH
+    file = tempfile.NamedTemporaryFile(suffix='.py', **kwds)
+    file.write(b(''.join([importable(object, alias=alias), name])))
+    file.flush()
+    return file
+
+
+def load(file, **kwds):
+    """load an object that was stored with dill.temp.dump
+
+    file: filehandle
+    mode: mode to open the file, one of: {'r', 'rb'}
+
+    >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5])
+    >>> dill.temp.load(dumpfile)
+    [1, 2, 3, 4, 5]
+    """
+    import dill as pickle
+    mode = kwds.pop('mode', 'rb')
+    name = getattr(file, 'name', file)  # name=file.name or name=file (if str)
+    return pickle.load(open(name, mode=mode, **kwds))
+
+
+def dump(object, **kwds):
+    """dill.dump of object to a NamedTemporaryFile.
+Loads with "dill.temp.load".  Returns the filehandle.
+
+    >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5])
+    >>> dill.temp.load(dumpfile)
+    [1, 2, 3, 4, 5]
+
+Optional kwds:
+    If 'suffix' is specified, the file name will end with that suffix,
+    otherwise there will be no suffix.
+    
+    If 'prefix' is specified, the file name will begin with that prefix,
+    otherwise a default prefix is used.
+    
+    If 'dir' is specified, the file will be created in that directory,
+    otherwise a default directory is used.
+    
+    If 'text' is specified and true, the file is opened in text
+    mode.  Else (the default) the file is opened in binary mode.  On
+    some operating systems, this makes no difference.
+
+NOTE: Keep the return value for as long as you want your file to exist !
+    """
+    import dill as pickle
+    import tempfile
+    file = tempfile.NamedTemporaryFile(**kwds)
+    pickle.dump(object, file)
+    file.flush()
+    return file
+
+
+def loadIO(buffer, **kwds):
+    """load an object that was stored with dill.temp.dumpIO
+
+    buffer: buffer object
+
+    >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5])
+    >>> dill.temp.loadIO(dumpfile)
+    [1, 2, 3, 4, 5]
+    """
+    import dill as pickle
+    if PY3:
+        from io import BytesIO as StringIO
+    else:
+        from StringIO import StringIO
+    value = getattr(buffer, 'getvalue', buffer)  # value or buffer.getvalue
+    if value != buffer: value = value()  # buffer.getvalue()
+    return pickle.load(StringIO(value))
+
+
+def dumpIO(object, **kwds):
+    """dill.dump of object to a buffer.
+Loads with "dill.temp.loadIO".  Returns the buffer object.
+
+    >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5])
+    >>> dill.temp.loadIO(dumpfile)
+    [1, 2, 3, 4, 5]
+    """
+    import dill as pickle
+    if PY3:
+        from io import BytesIO as StringIO
+    else:
+        from StringIO import StringIO
+    file = StringIO()
+    pickle.dump(object, file)
+    file.flush()
+    return file
+
+
+def loadIO_source(buffer, **kwds):
+    """load an object that was stored with dill.temp.dumpIO_source
+
+    buffer: buffer object
+    alias: string name of stored object
+
+    >>> f = lambda x:x**2
+    >>> pyfile = dill.temp.dumpIO_source(f, alias='_f')
+    >>> _f = dill.temp.loadIO_source(pyfile)
+    >>> _f(4)
+    16
+    """
+    alias = kwds.pop('alias', None)
+    source = getattr(buffer, 'getvalue', buffer)  # source or buffer.getvalue
+    if source != buffer: source = source()  # buffer.getvalue()
+    if PY3: source = source.decode()  # buffer to string
+    if not alias:
+        tag = source.strip().splitlines()[-1].split()
+        if tag[0] != '#NAME:':
+            stub = source.splitlines()[0]
+            raise IOError("unknown name for code: %s" % stub)
+        alias = tag[-1]
+    local = {}
+    exec(source, local)
+    _ = eval("%s" % alias, local)
+    return _
+
+
+def dumpIO_source(object, **kwds):
+    """write object source to a buffer (instead of dill.dump)
+Loads by with dill.temp.loadIO_source.  Returns the buffer object.
+
+    >>> f = lambda x:x**2
+    >>> pyfile = dill.temp.dumpIO_source(f, alias='_f')
+    >>> _f = dill.temp.loadIO_source(pyfile)
+    >>> _f(4)
+    16
+
+Optional kwds:
+    If 'alias' is specified, the object will be renamed to the given string.
+    """
+    from .source import importable, getname
+    if PY3:
+        from io import BytesIO as StringIO
+    else:
+        from StringIO import StringIO
+    alias = kwds.pop('alias', '')  # XXX: include an alias so a name is known
+    name = str(alias) or getname(object)
+    name = "\n#NAME: %s\n" % name
+    # XXX: assumes kwds['dir'] is writable and on $PYTHONPATH
+    file = StringIO()
+    file.write(b(''.join([importable(object, alias=alias), name])))
+    file.flush()
+    return file
+
+
+del contextlib
+
+# EOF

idmtools_test/inputs/python/folder_dup_file/model1.py

@@ -0,0 +1,19 @@
+import json
+import os
+import sys
+
+# dir_path is current file dir which is under "Assets" dir in COMPS
+dir_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dir_path, "MyExternalLibrary"))
+
+# create 'output' dir in COMPS under current working dir which is one dir above "Assets" dir
+output_dir = os.path.join(dir_path, "..", "output")
+if not os.path.exists(output_dir):
+    os.mkdir(output_dir)
+
+config = json.load(open(os.path.join(dir_path, "..", "config.json"), "r"))
+print(config)
+
+# write each configs to result.json in comps's simulation output
+with open(os.path.join(output_dir, "result.json"), "w") as fp:
+    json.dump(config, fp)

idmtools_test/inputs/python/hello_world.py

@@ -0,0 +1 @@
+print('Hello World!')

idmtools_test/inputs/python/model.py

@@ -0,0 +1,26 @@
+import os
+import sys
+
+CURRENT_DIRECTORY = os.path.dirname(__file__)
+LIBRARY_PATH = os.path.join(CURRENT_DIRECTORY, 'site-packages')  # Need to site-packages level!!!
+
+"""
+sys.path.insert(0, LIBRARY_PATH) will search packages from experiment's 'Assets/site-packages' first,
+then default HPC python site-packages.
+"astor" package does not exist in comps, so we use custom_lib to upload to experiment's Assets/site-packages directory
+Please go to examples/load_lib/example-load-lib.py to see how we upload 'astor' package to comps
+"""
+sys.path.insert(0, LIBRARY_PATH)
+sys.path.append(os.path.join(CURRENT_DIRECTORY))
+print(sys.path)
+from MyExternalLibrary.functions import add
+
+if __name__ == "__main__":
+    import json
+
+    with open("config.json", 'r') as fp:
+        config = json.load(fp)
+        parameters = config["parameters"]
+
+    print(add(parameters["a"], parameters["b"]))
+    print(config["parameters"])

idmtools_test/inputs/python/model1.py

@@ -0,0 +1,20 @@
+import json
+import os
+import sys
+
+current_dir = os.path.abspath(os.getcwd())
+
+# create 'output' dir in COMPS under current working dir which is one dir above "Assets" dir
+output_dir = os.path.join(current_dir, "output")
+config_path = os.path.join(current_dir, "config.json")
+if not os.path.exists(output_dir):
+    os.mkdir(output_dir)
+
+config = json.load(open(config_path, "r"))
+print(config)
+
+# write each configs to result.json in comps's simulation output
+with open(os.path.join(output_dir, "result.json"), "w") as fp:
+    json.dump(config, fp)
+
+sys.exit(0)

idmtools_test/inputs/python/model3.py

@@ -0,0 +1,21 @@
+import os
+
+from MyLib.functions import add
+current_dir = os.path.abspath(os.getcwd())
+if __name__ == "__main__":
+    import json
+
+    with open("config.json", 'r') as fp:
+        config = json.load(fp)
+        parameters = config["parameters"]
+
+    result = add(parameters["a"], parameters["b"])
+    print(result)
+    print(config["parameters"])
+
+    output_dir = os.path.join(current_dir, "output")
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+    with open(os.path.join(output_dir, "result.txt"), "w") as fp:
+        fp.write("result:")
+        fp.write(str(result))

idmtools_test/inputs/python/newmodel2.py

@@ -0,0 +1,20 @@
+import json
+import os
+import sys
+
+if __name__ == "__main__":
+    current_directory = os.path.abspath(os.getcwd())
+    with open("config.json", 'r') as fp:
+        config = json.load(fp)
+        run_number = config["a"]
+
+    os.makedirs(os.path.join(current_directory, "output"))
+    with open(os.path.join(current_directory, "output", "output.json"), 'w') as fp:
+        json.dump({"a": run_number}, fp)
+    print("Simulation running with Run_Number = " + str(run_number))
+
+    # The local platform needs to know the resulting status of a work item. We provide it through a return code
+    # print(f"Simulation running with Run_Number = {run_number}")
+    # 0 means succeeded. Any other value is an error. It is optional if you want to provide specific return codes
+    # for specific errors
+    sys.exit(0)

idmtools_test/inputs/python/output_generator/generate.py

@@ -0,0 +1,39 @@
+import argparse
+import random
+from logging import getLogger, basicConfig
+
+CHUNK_GENERATION_SIZE = 1048576 * 4
+basicConfig()
+logger = getLogger()
+
+
+def generate_chunk(filename: str, total_bytes: int):
+    logger.info(f"Writing chunk of {total_bytes} to {filename}")
+    with open(filename, 'wb') as out:
+        total_written = 0
+        while total_written < total_bytes:
+            # Write in 1 mb segments
+            if total_written + CHUNK_GENERATION_SIZE > total_bytes:
+                ws = total_bytes - total_written
+            else:
+                ws = CHUNK_GENERATION_SIZE
+            out.write(bytearray(random.getrandbits(8) for _ in range(ws)))
+            total_written += CHUNK_GENERATION_SIZE
+
+
+def generate_chunks(total_chunks: int, chunk_size: int, chunk_name: str):
+    logger.info(f"Writing {total_chunks} chunks")
+    for i in range(total_chunks):
+        fn = chunk_name.format(idx=i)
+        generate_chunk(fn, chunk_size)
+
+
+if __name__ == "__main__":
+    random.seed()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--chunk-size", default=1024 * 256, type=int)
+    parser.add_argument("--chunks", default=100, type=int)
+    parser.add_argument("--chunk-name", default="{idx:05d}.chunk")
+
+    args = parser.parse_args()
+    generate_chunks(args.chunks, args.chunk_size, args.chunk_name)

idmtools_test/inputs/python/realpath_verify.py

@@ -0,0 +1,6 @@
+import os
+
+# This file is used to verify consistent file paths for realpath across platforms
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+print(dir_path)