ellipsis 3.1.1__tar.gz → 3.3.0__tar.gz

{ellipsis-3.1.1 → ellipsis-3.3.0}/PKG-INFO

@@ -1,12 +1,10 @@
 Metadata-Version: 2.1
 Name: ellipsis
-Version: 3.1.1
+Version: 3.3.0
 Summary: Package to interact with the Ellipsis API
 Home-page: https://github.com/ellipsis-drive-internal/python-package
 Author: Daniel van der Maas
 Author-email: daniel@ellipsis-drive.com
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -50,5 +48,3 @@ Another example
     info = el.path.get(folderId, token)
     layers = el.path.listPath(folderId, pathType='layer', token = token, listAll = True)
     folders = el.path.listPath(folderId, pathType='folder', token = token, listAll = True)
-
-

{ellipsis-3.1.1 → ellipsis-3.3.0}/ellipsis/__init__.py

@@ -4,7 +4,9 @@
 from ellipsis import account
 from ellipsis import user
 from ellipsis import path
-from ellipsis import view
 from ellipsis import util
+from ellipsis import compute
+__version__ = '3.3.0'
+
+
 
-__version__ = '3.1.1'

ellipsis-3.3.0/ellipsis/account/__init__.py

@@ -0,0 +1,4 @@
+
+from ellipsis.account.root import logIn, listRoot, getInfo
+
+from ellipsis.account import accessToken

{ellipsis-3.1.1 → ellipsis-3.3.0}/ellipsis/account/root.py

@@ -1,6 +1,7 @@
 from ellipsis import apiManager
 from ellipsis import sanitize
 from ellipsis.util.root import recurse
+import requests
 
 def logIn(username, password, validFor = None):
 
@@ -10,12 +11,28 @@ def logIn(username, password, validFor = None):
 
         json = {'username': username, 'password': password, 'validFor': validFor}
 
+        r = apiManager.call(requests.post,'/account/login', body=json, token=None, crash=False)
+        if r.status_code == 400:
+            x = r.json()
+            if x['message'] == "No password configured.":
+                raise ValueError("You cannot login with your Google credentials in the Python module. You need to configure an Ellipsis Drive specific password. You can do this on https://app.ellipsis-drive.com/account-settings/security")
+        if r.status_code != 200:
+            raise ValueError(r.text)
 
-        r = apiManager.post('/account/login', json)
+        r = r.json()
         token = r['token']
 
         return(token)
 
+
+def getInfo(token):
+    token = sanitize.validString('token', token, True)
+
+    r = apiManager.get( '/account', body={}, token=token)
+
+
+    return r
+
 def listRoot(rootName, token, pathTypes= None, pageStart = None, listAll = True):
     token = sanitize.validString('token', token, True)
     rootName = sanitize.validString('rootName', rootName, True)

ellipsis-3.3.0/ellipsis/apiManager.py

@@ -0,0 +1,183 @@
+import requests
+import json
+import urllib
+import os
+import time
+from requests_toolbelt import MultipartEncoder
+
+baseUrl = 'https://api.ellipsis-drive.com/v3'
+
+
+def filterNone(body, toString= False):
+    if type(body) == type(None):
+        return body
+
+    params = {}
+    for k in body.keys():
+        if type(body[k]) != type(None):
+            if toString:
+                if str(type(body[k])) != str(type('x')):
+                    params[k] = json.dumps(body[k])
+                else:
+                    params[k] = body[k]
+                    
+            else:
+                params[k] = body[k]
+    return params
+
+def get(url, body = None, token = None, crash = True, parseJson = True):
+    if body == None:
+        body = {'token': token}
+    else:
+        body['token'] = token
+    body = filterNone(body)
+
+    for k in body.keys():
+        if type(body[k]) != type('x') :
+            body[k] = json.dumps(body[k])
+
+
+    body = urllib.parse.urlencode(body)
+
+    url = url + '?' + body
+
+    r = call( method = requests.get, url = url, body = None, token = token, crash = crash, parseJson = parseJson )
+
+
+    return r
+
+
+def post(url, body, token=None):
+    r = call(method=requests.post, url=url, body=body, token=token)
+    return r
+
+
+def put(url, body, token=None):
+    r = call(method=requests.put, url=url, body=body, token=token)
+    return r
+
+
+def patch(url, body, token=None):
+    r = call(method=requests.patch, url=url, body=body, token=token)
+    return r
+
+    
+def delete(url, body, token = None):
+    r = call( method = requests.delete, url = url, body = body, token = token )
+    return r
+
+RETRIES = 10
+WAIT = 2
+def call(method, url, body = None, token = None, crash = True, parseJson = True):
+    tried = 0
+    while tried <RETRIES:
+        try:
+            r = actualCall(method, url, body, token)
+            break
+        except Exception as ex:
+            print('time out detected, retrying request')
+            print('url', url)
+            print('error',ex)
+            time.sleep(WAIT)
+            tried = tried +1
+    if tried >= RETRIES:
+        raise ValueError('Could not reach server')
+    if crash:
+        if r.status_code != 200:
+            raise ValueError(r.text)
+
+        if parseJson:
+            try:
+                r = r.json()
+            except:
+                r = r.text
+
+        return r
+    else:
+        return r
+
+TIMEOUTTIME = 20
+def actualCall(method, url, body, token):
+    body = filterNone(body)
+    if type(body) != type(None) and type(body) != type({}):
+        raise ValueError(
+            'body of an API call must be of type dict or noneType')
+
+    if type(token) != type(None) and type(token) != type('x'):
+        raise ValueError('Token must be of type string or noneType')
+
+    if token == None:
+        r = method(baseUrl + url, json=body, timeout=TIMEOUTTIME)
+    else:
+        if not 'Bearer' in token:
+            token = 'Bearer ' + token
+        r = method(baseUrl + url, json=body, headers={"Authorization": token}, timeout=TIMEOUTTIME)
+
+    return r
+
+def upload(url, filePath, body, token, key = 'data', memfile= None):
+    body['debug'] = True
+    body = filterNone(body, toString=True)
+
+    seperator = os.path.sep
+    fileName = filePath.split(seperator)[len(filePath.split(seperator))-1 ]
+
+    if str(type(memfile)) == str(type(None)):
+        conn_file = open(filePath, 'rb')
+    else:
+        conn_file = memfile
+
+    payload = MultipartEncoder(fields = {**body, key: (fileName, conn_file, 'application/octet-stream')})
+    if not 'Bearer' in token:
+        token = 'Bearer ' + token
+
+    r = requests.post(baseUrl + url, headers = {"Authorization":token, "Content-Type": payload.content_type}, data=payload, verify=False)
+
+    if str(type(memfile)) == str(type(None)):
+        conn_file.close()
+    
+    if r.status_code != 200:
+        raise ValueError(r.text)
+    return r.json()
+
+def download(url, filePath=None, token = None, memfile = None):
+    if type(token) == type(None):
+        with requests.get(baseUrl + url, stream=True) as r:
+            r.raise_for_status()
+            if str(type(memfile)) == str(type(None)):
+                with open(filePath, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=8192): 
+                        # If you have chunk encoded response uncomment if
+                        # and set chunk_size parameter to None.
+                        #if chunk: 
+                        f.write(chunk)
+            else:
+                for chunk in r.iter_content(chunk_size=8192): 
+                    # If you have chunk encoded response uncomment if
+                    # and set chunk_size parameter to None.
+                    #if chunk: 
+                    memfile.write(chunk)
+    
+        
+        
+    else:
+        token = 'Bearer ' + token
+        with requests.get(baseUrl + url, stream=True, headers={"Authorization": token}) as r:
+            r.raise_for_status()
+            if str(type(memfile)) == str(type(None)):
+                with open(filePath, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=8192): 
+                        # If you have chunk encoded response uncomment if
+                        # and set chunk_size parameter to None.
+                        #if chunk: 
+                        f.write(chunk)
+            else:
+                for chunk in r.iter_content(chunk_size=8192): 
+                    # If you have chunk encoded response uncomment if
+                    # and set chunk_size parameter to None.
+                    #if chunk: 
+                    memfile.write(chunk)
+    
+
+    
+    

ellipsis-3.3.0/ellipsis/compute/__init__.py

@@ -0,0 +1 @@
+from ellipsis.compute.root import createCompute, listComputes, execute, terminateCompute, terminateAll, addToLayer

ellipsis-3.3.0/ellipsis/compute/root.py

@@ -0,0 +1,180 @@
+import dill
+import base64
+import time
+
+from ellipsis.util.root import recurse
+from ellipsis import sanitize
+from ellipsis.account import getInfo
+from ellipsis import apiManager
+from ellipsis.path.raster.timestamp.file import add as addFile
+from ellipsis.path.raster.timestamp import activate
+from io import BytesIO
+
+def createCompute(layers, token, files = None, nodes=None, interpreter='python3.12', requirements= [], awaitTillStarted = True, largeResult = False):
+    layers = sanitize.validDictArray('layers', layers, True)
+    files = sanitize.validUuidArray('files', files, False)
+    token = sanitize.validString('token', token, True)
+    nodes = sanitize.validInt('nodes', nodes, False)
+    interpreter = sanitize.validString('interpreter', interpreter, True)
+    requirements = sanitize.validStringArray('requirements', requirements, False)
+    largeResult = sanitize.validBool('largeResult', largeResult, True)
+    if type(nodes) == type(None):
+        info = getInfo(token=token)
+        nodes = info['plan']['maxComputeNodes']
+        if nodes == 0:
+            raise ValueError('You have no compute nodes in your plan. Please update your subscription')
+
+    requirements = "\n".join(requirements)
+
+    body = {'layers':layers, 'files':files, 'interpreter':interpreter, 'nodes':nodes, 'requirements':requirements, 'largeResult': largeResult}
+    r = apiManager.post('/compute', body, token)
+
+    computeId = r['id']
+    while awaitTillStarted:
+        print('waiting')
+        res = listComputes(token=token)['result']
+        r = [x for x in res if x['id'] == computeId][0]
+        if r['status'] == 'available':
+            break
+        if r['status'] == 'errored':
+            raise ValueError(r['message'])
+
+        time.sleep(1)
+
+    return {'id':computeId}
+
+
+def execute(computeId, f, token, awaitTillCompleted=True, writeToLayer = None):
+    computeId = sanitize.validUuid('computeId', computeId, True)
+    token = sanitize.validString('token', token, True)
+    writeToLayer = sanitize.validObject('writeToLayer', writeToLayer, False)
+    awaitTillCompleted = sanitize.validBool('awaitTillCompleted', awaitTillCompleted, False)
+
+    if str(type(f)) != "<class 'function'>":
+        raise ValueError('parameter f must be a function')
+    if type(writeToLayer) != type(None):
+        if not 'file' in writeToLayer:
+            writeToLayer['file'] = {'format':'tif'}
+    f_bytes = dill.dumps(f)
+    f_string = base64.b64encode( f_bytes )
+    f_string = str(f_string)[2: -1]
+    body = { 'file':f_string, 'writeToLayer':writeToLayer}
+    apiManager.post('/compute/' + computeId + '/execute', body, token)
+
+    while awaitTillCompleted:
+        res = listComputes(token=token)['result']
+        r = [x for x in res if x['id'] == computeId][0]
+        print('waiting')
+        if r['status'] == 'completed':
+            break
+        if r['status'] == 'errored':
+            raise ValueError(str(r['message']))
+        time.sleep(1)
+
+    for x in r['result']:
+        if x['type'] == 'exception':
+            raise x['value']
+
+
+    values = [ '/compute/' + computeId + '/file/'+ x['value'] if x['type'] == 'file' else x['value'] for x in r['result']]
+    return values
+
+def parseResults(r):
+    results = []
+    for x in r:
+        x = base64.b64decode(x)
+        x = dill.loads(x)
+        results = results + x
+
+    return results
+
+def terminateCompute(computeId, token, awaitTillTerminated = True):
+    computeId = sanitize.validUuid('computeId', computeId, True)
+    token = sanitize.validString('token', token, True)
+    sanitize.validBool('awaitTillTerminated',awaitTillTerminated, True)
+
+
+    r = apiManager.post('/compute/' + computeId + '/terminate', {}, token)
+
+    while awaitTillTerminated:
+        res = listComputes(token=token)['result']
+        z = [x for x in res if x['id'] == computeId][0]
+        if z['status'] == 'stopped':
+            break
+        time.sleep(1)
+
+    return r
+
+def terminateAll(token, awaitTillTerminated = True ):
+    token = sanitize.validString('token', token, True)
+    sanitize.validBool('awaitTillTerminated',awaitTillTerminated, True)
+
+    res = listComputes(token = token)['result']
+
+    for x in res:
+        if x['status'] != 'stopped' and x['status'] != 'errored' and x['status'] != 'stopping':
+
+            apiManager.post('/compute/' + x['id'] + '/terminate', {}, token)
+
+            while awaitTillTerminated:
+                res = listComputes(token=token)['result']
+                z = [x for x in res if x['id'] == x['id']][0]
+                if z['status'] == 'stopped':
+                    break
+                time.sleep(1)
+
+
+
+def getComputeInfo(computeId, token):
+    res = listComputes(token=token)['result']
+    r = [x for x in res if x['id'] == computeId]
+    if len(r) ==0:
+        raise ValueError('No compute found for given id')
+    return r[0]
+
+def listComputes(token, pageStart = None, listAll = True):
+    token = sanitize.validString('token', token, True)
+
+
+    body = { 'pageStart':pageStart }
+
+
+    def f(body):
+        return apiManager.get('/compute', body, token)
+
+    r = recurse(f, body, listAll)
+    for i in range(len(r['result'])):
+        if 'result' in r['result'][i]:
+                r['result'][i]['result'] = parseResults(r['result'][i]['result'])
+    return r
+
+def addToLayer(response, pathId, timestampId, token):
+    pathId = sanitize.validUuid('pathId', pathId, True)
+    timestampId = sanitize.validUuid('timestampId', timestampId, True)
+    token = sanitize.validString('token', token, True)
+
+    for url in response:
+        print('fetching file ' + url.split('/')[-1])
+        memfile = BytesIO()
+        memfile = downloadFile(url,  token, memfile=memfile)
+        print('read file ' + url.split('/')[-1])
+        print('adding file ' + url.split('/')[-1])
+        addFile(pathId = pathId, timestampId=timestampId, token = token, fileFormat='tif', memFile=memfile, name= url.split('/')[-1] + '.tif' )
+        print('file ' + url.split('/')[-1] + ' added to layer')
+    activate(pathId=pathId, timestampId=timestampId, token=token)
+    print('layer can now be found at ' + apiManager.baseUrl + '/drive/me?pathId=' + pathId )
+
+
+
+def downloadFile(url,  token, filePath = None, memfile = None):
+
+    url = sanitize.validString('url', url, True)
+    token = sanitize.validString('token', token, True)
+    filePath = sanitize.validString('filePath', filePath, False)
+    if memfile == None and filePath == None:
+        raise ValueError('Either memfile or filePath is required')
+
+    apiManager.download(url = url, filePath=filePath, memfile=memfile, token = token)
+
+    if memfile != None:
+        return memfile

{ellipsis-3.1.1 → ellipsis-3.3.0}/ellipsis/path/__init__.py

@@ -1,10 +1,13 @@
-from ellipsis.path.root import searchFolder, searchFile, searchRaster, favorite, unfavorite, editPublicAccess, searchVector, delete, editMetadata, move, rename, trash, recover, get
+from ellipsis.path.root import search, favorite, unfavorite, editPublicAccess, delete, editMetadata, move, rename, trash, recover, get
 from ellipsis.path import hashtag
 from ellipsis.path import invite
 from ellipsis.path import member
 from ellipsis.path import raster
 from ellipsis.path import vector
+from ellipsis.path import pointCloud
 from ellipsis.path import usage
 from ellipsis.path import folder
 from ellipsis.path import file
+from ellipsis.path import bookmark
+from ellipsis.path import setUpTask
 

ellipsis-3.3.0/ellipsis/path/bookmark/__init__.py

@@ -0,0 +1,2 @@
+
+from ellipsis.path.bookmark.root import add, get, edit

ellipsis-3.3.0/ellipsis/path/bookmark/root.py

@@ -0,0 +1,39 @@
+from ellipsis import apiManager
+from ellipsis import sanitize
+
+
+def get(pathId, token=None):
+    token = sanitize.validString('token', token, False)
+    pathId = sanitize.validUuid('pathId', pathId, True)
+
+    r = apiManager.get('/path/' + pathId + '/bookmark', {}, token)
+        
+    return r
+
+
+
+def add(name, bookmark, token, parentId = None, publicAccess = None, metadata=None):
+    token = sanitize.validString('token', token, True)
+    bookmark = sanitize.validObject('bookmark', bookmark, True)
+    name = sanitize.validString('pathId', name, False)
+    metadata = sanitize.validObject('metadata', metadata, False)
+    publicAccess = sanitize.validObject('publicAccess', publicAccess, False)
+    parentId = sanitize.validUuid('parentId', parentId, False)
+
+    r = apiManager.post('/path/bookmark', {'name':name, 'bookmark':bookmark , 'parentId':parentId, 'publicAccess':publicAccess, 'metadata':metadata}, token)
+        
+    return r
+
+
+def edit(pathId, token, layers=None, dems=None):
+    layers = sanitize.validObject('layers', layers, False)
+    dems = sanitize.validObject('dems', dems, False)
+    pathId = sanitize.validUuid('pathId', pathId, True)
+
+    r = apiManager.patch('/path/' + pathId + '/bookmark', {'layers':layers, 'dems':dems}, token)
+        
+    return r
+
+
+
+

ellipsis-3.3.0/ellipsis/path/file/__init__.py

@@ -0,0 +1,2 @@
+from ellipsis.path.file.root import download, add, addPickle, getPickle, addCsv, getCsv, addJson, getJson
+

ellipsis-3.3.0/ellipsis/path/file/root.py

@@ -0,0 +1,155 @@
+from ellipsis import apiManager
+from ellipsis import sanitize
+
+import os
+import pickle
+from io import BytesIO
+import json
+import pandas as pd
+
+
+def add(  token, filePath=None, memFile=None, parentId = None, publicAccess =None, metadata=None,name=None):
+    token = sanitize.validString('token', token, True)
+    parentId = sanitize.validUuid('parentId', parentId, False)
+    publicAccess = sanitize.validObject('publicAccess', publicAccess, False)
+    metadata = sanitize.validObject('metadata', metadata, False)
+    filePath = sanitize.validString('filePath', filePath, False)
+    name = sanitize.validString('name', name, False)
+    if type(memFile) == type(None) and type(filePath) == type(None):
+        raise ValueError('You need to specify either a filePath or a memFile')
+    if type(memFile) != type(None) and type(name) == type(None):
+        raise ValueError('Parameter name is required when using a memory file')
+
+    seperator = os.path.sep    
+    fileName = filePath.split(seperator)[len(filePath.split(seperator))-1 ]
+    if len(fileName) > 64:
+        fileName = fileName[0:63]
+    body = {'name':fileName, 'publicAccess':publicAccess, 'metadata':metadata, 'parentId':parentId}
+    if type(memFile) == type(None):
+        r = apiManager.upload('/path/file' , filePath, body, token, key = 'data')
+    else:
+        r = apiManager.upload('/path/file' , filePath , name, body, token, memfile = memFile)
+
+    return r
+
+
+def download(pathId, filePath, token=None):
+    pathId = sanitize.validUuid('pathId', pathId, True)
+    token = sanitize.validString('token', token, False)
+    filePath = sanitize.validString('filePath', filePath, True)
+
+
+    apiManager.download('/path/' + pathId + '/file/data', filePath, token)
+    
+
+
+def addCsv(df, name, token, parentId = None, publicAccess =None, metadata=None):
+    token = sanitize.validString('token', token, True)
+    name = sanitize.validString('name', name, True)
+    df = sanitize.validPandas('df', df, True)
+
+    parentId = sanitize.validUuid('parentId', parentId, False)
+    publicAccess = sanitize.validObject('publicAccess', publicAccess, False)
+    metadata = sanitize.validObject('metadata', metadata, False)
+    
+    memfile = BytesIO()
+    df.to_csv(memfile)
+    
+
+    body = {'name':name, 'publicAccess':publicAccess, 'metadata':metadata, 'parentId':parentId}
+    r = apiManager.upload('/path/file' , name,  body, token, key='data', memfile = memfile)
+
+    
+    return r
+
+def getCsv(pathId, token=None):
+    pathId = sanitize.validUuid('pathId', pathId, True)
+    token = sanitize.validString('token', token, False)
+
+
+    r = apiManager.get('/path/' + pathId + '/file/data', {}, token)
+
+
+    memfile = BytesIO(bytes(r, 'utf-8'))
+    try:
+        df =  pd.read_csv(memfile)
+    except:
+        raise ValueError('Read file is not a valid CSV')
+    return df
+
+
+
+def addJson(d, name, token, parentId = None, publicAccess =None, metadata=None):
+    token = sanitize.validString('token', token, True)
+    name = sanitize.validString('name', name, True)
+    d = sanitize.validObject('d', d, True)
+
+    parentId = sanitize.validUuid('parentId', parentId, False)
+    publicAccess = sanitize.validObject('publicAccess', publicAccess, False)
+    metadata = sanitize.validObject('metadata', metadata, False)
+    
+    memfile = BytesIO()
+    d = json.dumps(d)
+    memfile.write(bytes(d, 'utf-8'))
+    
+
+    body = {'name':name, 'publicAccess':publicAccess, 'metadata':metadata, 'parentId':parentId}
+    r = apiManager.upload('/path/file' , name,  body, token, key='data', memfile = memfile)
+
+    
+    return r
+
+
+def getJson(pathId, token=None):
+    pathId = sanitize.validUuid('pathId', pathId, True)
+    token = sanitize.validString('token', token, False)
+
+    memfile = BytesIO()
+    apiManager.download('/path/' + pathId + '/file/data', '', token, memfile)    
+    memfile.seek(0)
+    try:
+        x = json.load(memfile)
+    except:
+        raise ValueError('Read file not a valid pickle file')
+
+
+    return x
+
+
+
+def addPickle(x, name, token, parentId = None, publicAccess =None, metadata=None):
+    token = sanitize.validString('token', token, True)
+    name = sanitize.validString('name', name, True)
+    parentId = sanitize.validUuid('parentId', parentId, False)
+    publicAccess = sanitize.validObject('publicAccess', publicAccess, False)
+    metadata = sanitize.validObject('metadata', metadata, False)
+    
+    memfile = BytesIO()
+    pickle.dump(x, memfile)
+
+    body = {'name':name, 'publicAccess':publicAccess, 'metadata':metadata, 'parentId':parentId}
+    r = apiManager.upload('/path/file' , name,  body, token, key='data', memfile = memfile)
+
+    return r
+    
+
+def getPickle(pathId, token=None):
+    pathId = sanitize.validUuid('pathId', pathId, True)
+    token = sanitize.validString('token', token, False)
+
+    memfile = BytesIO()
+    apiManager.download('/path/' + pathId + '/file/data', '', token, memfile)    
+    memfile.seek(0)
+    try:
+        x = pickle.load(memfile)
+    except:
+        raise ValueError('Read file not a valid pickle file')
+
+
+    return x
+
+
+
+
+
+

{ellipsis-3.1.1 → ellipsis-3.3.0}/ellipsis/path/folder/__init__.py

@@ -1,2 +1,4 @@
 from ellipsis.path.folder.root import listFolder
 from ellipsis.path.folder.root import add
+from ellipsis.path.folder.root import traverse
+

{ellipsis-3.1.1 → ellipsis-3.3.0}/ellipsis/path/folder/root.py

@@ -36,3 +36,17 @@ def add( name, token, parentId = None, publicAccess =None, metadata=None):
 
     return apiManager.post('/path/folder', body, token)
 
+def traverse(pathId, location, pathType, token):
+    pathId = sanitize.validUuid('pathId', pathId, True)
+    location = sanitize.validStringArray('location', location, True)
+    pathType = sanitize.validString('pathType', pathType, True)
+    token = sanitize.validString('token', token, False)
+
+    body = {'location': location, 'type':pathType }
+
+    return apiManager.get('/path/' + pathId + '/folder/traverse', body, token)
+
+
+
+
+