basepair 1.5__0-py3.8.egg

basepair/api.py

@@ -0,0 +1,1353 @@
+'''
+Python interface to Basepair's REST API
+
+Copyright [2015] - [2018] Basepair INC
+All Rights Reserved.
+
+NOTICE: All information contained herein is, and remains
+the property of Basepair INC and its suppliers, if any.
+The intellectual and technical concepts contained
+herein are proprietary to Basepair INC
+and its suppliers and may be covered by U.S. and Foreign Patents,
+patents in process, and are protected by trade secret or copyright law.
+Dissemination of this information or reproduction of this material
+is strictly forbidden unless prior written permission is obtained
+from Basepair INC.
+'''
+
+# pylint: disable=too-many-lines
+
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import json
+import subprocess
+from subprocess import CalledProcessError
+import time
+import datetime
+
+# App imports
+from .helpers import eprint, NicePrint, SetFilter
+from .infra.configuration import Parser
+from .infra.webapp import Analysis, File, Gene, Genome, GenomeFile, Host, Module, Pipeline, Project, Sample, Upload, User
+
+class BpApi(): # pylint: disable=too-many-instance-attributes,too-many-public-methods
+  ''' A wrapper over the REST API for accessing the Basepair system
+
+  Use it thus:
+
+  > import basepair
+  > bp = basepair.connect()
+
+  conf is a JSON object with at least these keys:
+  {
+      "api": {
+          "host": "app.basepairtech.com",
+          "prefix": "/api/v2/",
+          "username": "utk",
+          "key": "xxx"
+      },
+  }
+
+  Some of the common methods are:
+
+  # create a sample
+  sample_id = bp.create_sample(info_dict)
+
+  #  create an analysis
+  analysis_id = bp.create_analysis(workflow_id, sample_id)
+
+  # download results - bam, dedup file
+  filename = bp.get_file_by_tags(sample_id, ['bam', 'dedup'])
+
+  # delete sample
+  bp.delete_sample(sample_id)
+  '''
+
+  def __init__(self, conf=None, scratch='.', use_cache=False, user_cache_for_host_conf=False, verbose=None): # pylint: disable=too-many-arguments
+    self.verbose = verbose
+
+    if conf:
+      self.conf = conf
+    elif 'BP_CONFIG_FILE' in os.environ:
+      self.conf = json.load(open(os.environ['BP_CONFIG_FILE']))
+    else:
+      if 'BP_USERNAME' not in os.environ:
+        eprint('ERROR: BP_USERNAME not set in env')
+        sys.exit(1)
+      if 'BP_API_KEY' not in os.environ:
+        eprint('ERROR: BP_API_KEY not set in env')
+        sys.exit(1)
+      username = os.environ['BP_USERNAME']
+      api_key = os.environ['BP_API_KEY']
+      self.conf = {
+        'api': {
+          'key': api_key,
+          'host': 'app.basepairtech.com',
+          'prefix': '/api/v2/',
+          'ssl': True,
+          'username': username,
+        }
+      }
+
+    if self.conf.get('api', {}).get('ssl') is None:
+      eprint('ERROR: The config file need to be updated. Please visit:')
+      eprint('https://test.basepairtech.com/api/v2/users/api_key')
+      eprint('To get your new config file.')
+      sys.exit(1)
+
+    self.scratch = self.conf.get('scratch', scratch).rstrip('/')
+    self.use_cache = use_cache
+
+    if use_cache and self.verbose == 1:
+      eprint('Warning: caching data.')
+
+    self._get_user_id()
+
+    self.genomes = self.get_genomes()
+
+    # Get configuration from host only if it is not set in the incoming config
+    cache = False
+    configuration = self.conf
+    if user_cache_for_host_conf:
+      eprint('INFO: Use cached host cloud service configuration.')
+      cache = '{}/json/config.json'.format(self.scratch)
+
+    if self.conf.get('api', {}).get('cli'):
+      configuration = (User(self.conf.get('api'))).get_configuration(cache=cache)
+    self.configuration = Parser(configuration)
+
+    self.DATATYPES = ['dna-seq', 'chip-seq', 'rna-seq', 'atac-seq', 'other'] # pylint: disable=invalid-name
+    self.LIST_TYPES = ['samples', 'analyses', 'genomes', 'workflows', 'analysis'] # pylint: disable=invalid-name
+
+  ################################################################################################
+  ### ANALYSIS ###################################################################################
+  ################################################################################################
+  def create_analysis(
+    self,
+    workflow_id,
+    control_id=None,
+    control_ids=[],
+    ignore_validation_warnings=False,
+    params=None,
+    project_id=None,
+    sample_id=None,
+    sample_ids=[],
+  ): # pylint: disable=dangerous-default-value,too-many-arguments
+    '''Create analysis
+    Parameters
+    ----------
+    workflow_id:                {str}   Workflow id. Run bp.get_workflows() to see what is available.  [Required]
+
+    control_id:                 {str}   Control sample id.
+    control_ids:                {list}  Control sample id.
+    ignore_validation_warnings: {bool}  Ignore validation warnings
+    params:                     {dict}  Dictionary of parameter values.
+    project_id:                 {str}   Project id.
+    sample_id:                  {str}   Sample id.
+    sample_ids:                 {list}  List of sample id.
+    '''
+
+    # get api version
+    prefix = self.conf.get('api', {}).get('prefix', '/api/v2/')
+    if control_id:
+      control_ids.append(control_id)
+    if sample_id:
+      sample_ids.append(sample_id)
+
+    # check if valid workflow id
+    if not self._check_workflow(workflow_id):
+      return None
+
+    # check if all sample ids are valid
+    if not all([self._check_sample(item_id) for item_id in sample_ids]):
+      return None
+
+    data = {
+      'controls': self._parsed_sample_list(control_ids, prefix),
+      'samples': self._parsed_sample_list(sample_ids, prefix),
+      'ignore_validation_warning': ignore_validation_warnings,
+      'workflow': '{}pipelines/{}'.format(prefix, workflow_id)
+    }
+
+    if project_id:
+      data['projects'] = ['{}projects/{}'.format(prefix, project_id)]
+    eprint(json.dumps(data, indent=2))
+
+    if params:
+      data['params'] = params
+
+    analysis_api = Analysis(self.conf.get('api'))
+    info = analysis_api.save(payload=data)
+    analysis_id = info.get('id')
+    if self.verbose and analysis_id:
+      eprint('created: analysis {} with sample id(s) {}'.format(
+          analysis_id,
+          ','.join(sample_ids),
+      ))
+    return analysis_id
+
+  def delete_analysis(self, uid):
+    '''Delete method'''
+    return (Analysis(self.conf.get('api'))).delete(uid)
+
+  def download_analysis(self, uid, outdir='.', tagkind=None, tags=None):
+    '''
+    Download files from one or more analysis.
+    Parameters
+    ----------
+    uid:     {int}  One or more uids identifying the analysesself             [Required]
+    outdir:  {str}  Output directory to download results to
+    tagkind: {str}  Type of tag filtering to do. Options: exact, diff, subset
+    tags:    {list} List of list of tags to filter files by
+    '''
+    if tagkind not in ['exact', 'diff', 'subset']:
+      eprint('Invalid tagkind, choose one of: exact, diff, subset')
+      return None
+    if tags is not None:
+      is_not_valid = not (isinstance(tags, list) and isinstance(tags[0], list))
+      if is_not_valid:
+        eprint('Invald tags argument. Provide a list of list of tags.')
+        return None
+    if not isinstance(uid, list):
+      uid = [uid]
+    for item_id in uid:
+      analysis = self.get_analysis(item_id)
+      self.get_analysis_files(
+        analysis=analysis,
+        dirname=outdir,
+        kind=tagkind,
+        tags=tags,
+      )
+    return None
+
+  def fusionsalysis(
+    self,
+    workflow_id,
+    control_id=None,
+    control_ids=[],
+    params=None,
+    sample_id=None,
+    sample_ids=[]
+  ): # pylint: disable=dangerous-default-value,too-many-arguments
+    '''Create analysis
+    Parameters
+    ----------
+    Parameters
+    ----------
+    workflow_id:                {str}   Workflow id. Run bp.get_workflows() to see what is available.  [Required]
+
+    control_id:                 {str}   Control sample id.
+    control_ids:                {list}  Control sample id.
+    params:                     {dict}  Dictionary of parameter values.
+    sample_id:                  {str}   Sample id.
+    sample_ids:                 {list}  List of sample id.
+    '''
+
+    prefix = self.conf.get('api', {}).get('prefix', '/api/v2/')
+    if control_id:
+      control_ids.append(control_id)
+    if sample_id:
+      sample_ids.append(sample_id)
+
+    data = {
+      'controls': self._parsed_sample_list(control_ids, prefix),
+      'samples': self._parsed_sample_list(sample_ids, prefix),
+      'workflow': '{}pipelines/{}'.format(prefix, workflow_id),
+    }
+
+    if params:
+      data['params'] = params
+
+    info = (Analysis(self.conf.get('api'))).save(payload=data)
+    if info.get('error'):
+      eprint('failed:', ','.join(sample_ids), info.get('msg'))
+      return None
+
+    analysis_id = info.get('id')
+    if self.verbose and analysis_id:
+      eprint('created: analysis {} w/ sample {}'.format(analysis_id, ','.join(sample_ids)))
+    return analysis_id
+
+  def get_analysis(self, uid):
+    '''Get resource'''
+    return (Analysis(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/analysis.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  def get_analysis_owner(self, analysis_id):
+    '''get owner user for analysis'''
+    user_id = self._get_analysis_owner_id(analysis_id)
+    return self.get_user(user_id) if user_id else None
+
+  def get_analyses(self):
+    '''Get resource list'''
+    return (Analysis(self.conf.get('api'))).list_all()
+
+  def update_analysis(self, uid, data):
+    '''Update resource'''
+    info = (Analysis(self.conf.get('api'))).save(
+        obj_id=uid,
+        payload=data,
+    )
+    if info.get('error'):
+      eprint('cudnt update analysis {}, msg {}'.format(uid, info.get('msg')))
+    if self.verbose:
+      eprint('analysis', uid, 'updated')
+    return info
+
+  ################################################################################################
+  ### FILE #######################################################################################
+  ################################################################################################
+  def create_file(self, uid, data):
+    '''Create resource'''
+    return (File(self.conf.get('api'))).save(obj_id=uid, payload=data)
+
+  def get_file(self, uid):
+    '''Get resource'''
+    return (File(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/file.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  def update_file(self, uid, data):
+    '''Update resource'''
+    return (File(self.conf.get('api'))).save(obj_id=uid, payload=data)
+
+  ################################################################################################
+  ### GENE #######################################################################################
+  ################################################################################################
+  def create_gene(self, data):
+    '''Create resource'''
+    return (Gene(self.conf.get('api'))).save(payload=data)
+
+  def delete_gene(self, uid):
+    '''Delete method'''
+    return (Gene(self.conf.get('api'))).delete(uid)
+
+  def get_gene(self, uid):
+    '''Get resource'''
+    return (Gene(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/gene.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  def get_genes(self):
+    '''Get resource list'''
+    info = (Gene(self.conf.get('api'))).list(
+        params={'limit': 0}
+    )
+    return info.get('objects', [])
+
+  def get_genes_by_info(self, genome=None, symbol=None, tx_id=None):
+    '''Get genes by info'''
+    params = {'limit': 0}
+    if genome:
+      params['genome__name'] = genome
+    if symbol:
+      params['symbol__iexact'] = symbol
+    if tx_id:
+      params['tx_id'] = tx_id
+    info = (Gene(self.conf.get('api'))).list(params)
+    return info.get('objects', [])
+
+  def update_gene(self, uid, data):
+    '''Update resource'''
+    info = (Gene(self.conf.get('api'))).save(obj_id=uid, payload=data)
+    if self.verbose and not info.get('error'):
+      eprint('gene', uid, 'updated')
+    return info
+
+  ################################################################################################
+  ### GENOME #####################################################################################
+  ################################################################################################
+  def create_genome(self, data):
+    '''Create resource'''
+    return (Genome(self.conf.get('api'))).save(payload=data)
+
+  def delete_genome(self, uid):
+    '''Delete method'''
+    return (Genome(self.conf.get('api'))).delete(uid)
+
+  def get_genome(self, uid):
+    '''Get resource'''
+    return (Genome(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/genome.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  def get_genomes(self):
+    '''Get resource list'''
+    return (Genome(self.conf.get('api'))).list_all()
+
+  def update_genome(self, uid, data):
+    '''Update resource'''
+    info = (Genome(self.conf.get('api'))).save(obj_id=uid, payload=data)
+    if self.verbose and not info.get('error'):
+      eprint('genome', uid, 'updated')
+    return info
+
+  ################################################################################################
+  ### GENOME FILE ################################################################################
+  ################################################################################################
+  def get_genomefile_by_filters(self, filters=None):
+    '''Get genome files by filters'''
+    if not filters:
+      eprint('Filters required.')
+      return None
+    filters['limit'] = 0
+    info = (GenomeFile(self.conf.get('api'))).list(params=filters)
+    return info.get('objects', [])
+
+  ################################################################################################
+  ### HOST #######################################################################################
+  ################################################################################################
+  def get_host_by_domain(self, domain):
+    '''Get host by domain'''
+    info = (Host(self.conf.get('api'))).list({
+        'domain__exact': domain,
+        'limit': 1,
+    })
+    return info.get('objects')[0] if info.get('objects') else None
+
+  ################################################################################################
+  ### MODULE #####################################################################################
+  ################################################################################################
+  def get_module(self, uid):
+    '''Get resource'''
+    return (Module(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/module.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  ################################################################################################
+  ### PIPELINE / WORKFLOW ########################################################################
+  ################################################################################################
+  def get_workflow(self, uid):
+    '''Get resource'''
+    return (Pipeline(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/workflow.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  def get_workflows(self):
+    '''Get resource list'''
+    return (Pipeline(self.conf.get('api'))).list_all()
+
+  ################################################################################################
+  ### PROJECT ####################################################################################
+  ################################################################################################
+  def create_project(self, data):
+    '''Create resource'''
+    info = (Project(self.conf.get('api'))).save(payload=data)
+    if self.verbose == 2:
+      eprint('Creating project:')
+      eprint('Data:', data)
+
+    project_id = info.get('id', None)
+    if project_id:  # success
+      if self.verbose:
+        eprint('created: project with id', project_id)
+    else:  # failure
+      eprint('failed project creation:', data['name'], info.get('msg'))
+    return project_id
+
+  def update_project(self, uid, data, params=None):
+    '''Update resource'''
+    info = (Project(self.conf.get('api'))).save(
+        obj_id=uid,
+        payload=data,
+        params=params
+    )
+    if self.verbose and not info.get('error'):
+      eprint('project', uid, 'updated')
+    return info
+
+  ################################################################################################
+  ### SAMPLE #####################################################################################
+  ################################################################################################
+  def create_sample(self, data, source='api', upload=True): # pylint: disable=too-many-branches,too-many-locals,too-many-statements
+    '''Create sample with the provided info
+
+    Parameters
+    ----------
+    data : dict
+        Dictionary of sample information.
+
+    source : str
+        TODO
+
+    upload : bool
+        Whether to upload the sample to the server or not.
+
+    '''
+
+    # get api version
+    prefix = self.conf.get('api', {}).get('prefix', '/api/v2/')
+
+    # some input validation
+    data['genome'] = self._get_genome_by_name(data.get('genome'))
+
+    if data.get('default_workflow'):
+      if not self._check_workflow(data['default_workflow']):
+        del data['default_workflow']
+        eprint('Provided workflow is not valid.')
+
+    if 'filepaths1' in data and data['filepaths1'] is None:
+      if self.verbose:
+        eprint('filepaths1 is None.')
+      del data['filepaths1']
+
+    if 'filepaths2' in data and data['filepaths2'] is None:
+      if self.verbose:
+        eprint('filepaths2 is None.')
+      del data['filepaths2']
+
+    # if only one sample as str, but into list
+    if data.get('filepaths1') and isinstance(data['filepaths1'], str):
+      data['filepaths1'] = [data['filepaths1']]
+
+    if data.get('filepaths2') and isinstance(data['filepaths2'], str):
+      data['filepaths2'] = [data['filepaths2']]
+
+    if data.get('filepaths2') and not data.get('filepaths1'):
+      eprint('ERROR: Parameter filepaths1 cannot be empty if filepaths2 is specified.')
+      return None
+
+    # validate unique file between filepaths1 and filepaths2
+    does_repeast = data.get('filepaths2') \
+      and any(path in data.get('filepaths2') for path in data.get('filepaths1'))
+    if does_repeast:
+      eprint('ERROR: Same file cannot be use in filepaths1 and filepaths2.')
+      return None
+
+    if data.get('default_workflow'):
+      data['default_workflow'] = '{}pipelines/{}'.format(prefix, data['default_workflow'])
+
+    if data.get('projects'):
+      project = (Project(self.conf.get('api'))).get(data.get('projects'))
+      if project.get('error'):
+        eprint('ERROR: Looking for project. {}'.format(project.get('msg')))
+        return None
+      data['projects'] = [project.get('resource_uri')]
+
+    info = (Sample(self.conf.get('api'))).save(payload=data)
+    sample_id = info.get('id')
+
+    if self.verbose == 2:
+      eprint('Creating sample:')
+      eprint('Data:', data)
+
+    if sample_id:  # success
+      if self.verbose:
+        eprint('created: sample with id', sample_id)
+    else:  # failure
+      eprint('failed sample creation:', data['name'], info.get('msg'))
+      return None
+
+    # do the actual upload, update filepath
+    files_to_upload = []
+    for path in data.get('filepaths1', []):
+      path = os.path.abspath(os.path.expanduser(
+          os.path.expandvars(path)
+      ))
+    for path in data.get('filepaths2', []):
+      path = os.path.abspath(os.path.expanduser(
+          os.path.expandvars(path)
+      ))
+
+    # if a sample id exists, then upload the files
+    if self.verbose:
+      eprint('Sample id: {}'.format(sample_id))
+
+    order = 0
+    all_files = data.get('filepaths1', []) + data.get('filepaths2', [])
+    for filepath in all_files:
+      if self.verbose:
+        eprint('Creating upload {}'.format(filepath))
+
+      upload_id, filepath, key = self.create_upload(
+          sample_id,
+          filepath,
+          order,
+          is_paired_end=filepath in data.get('filepaths2', []),
+          source=source
+      )
+      files_to_upload.append([upload_id, filepath, key])
+      order += 1
+
+    if upload:
+      for upload_id, filepath, key in files_to_upload:
+        if self.verbose:
+          eprint('Uploading. upload_id: {}, filepath: {}, key: {}'.format(
+              upload_id,
+              filepath,
+              key
+          ))
+        self.upload_file(upload_id, filepath, key)
+    return sample_id
+
+  def delete_sample(self, uid):
+    '''Delete method'''
+    info = (Sample(self.conf.get('api'))).delete(uid)
+    if info.get('error'):
+      eprint('error: deleting {}, msg {}'.format(uid, info.get('msg')))
+    if self.verbose:
+      eprint('deleted sample', uid)
+    return info
+
+  def get_sample(self, uid, add_analysis=True):
+    '''Get resource'''
+    info = (Sample(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/sample.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+    if info and add_analysis:
+      info = self._add_full_analysis(info)
+    return info
+
+  def get_sample_owner(self, sample_id):
+    '''Get sample owner'''
+    user_id = self._get_sample_owner_id(sample_id)
+    return self.get_user(user_id) if user_id else None
+
+  def get_samples(self):
+    '''Get resource list'''
+    return (Sample(self.conf.get('api'))).list_all()
+
+  def sample_name_to_id(self, name):
+    '''Get sample id from name'''
+    response = (Sample(self.conf.get('api'))).list({'limit': 2, 'name': name})
+    items = response.get('objects')
+    if not items:
+      eprint('WARNING: no sample by name', name)
+      return None
+    if len(items) > 1:
+      eprint('WARNING: multiple sample by name', name)
+    return items[0].get('id')
+
+  def update_sample(self, uid, data):
+    '''Update resource'''
+    info = (Sample(self.conf.get('api'))).save(obj_id=uid, payload=data)
+    if self.verbose and not info.get('error'):
+      eprint('sample', uid, 'updated')
+    return info
+
+  ################################################################################################
+  ### UPLOAD #####################################################################################
+  ################################################################################################
+  def create_upload(self, sample_id, filepath, order, is_paired_end, source='api', uri=None): # pylint: disable=too-many-arguments
+    '''Create a upload object and actually upload the file'''
+    prefix = self.conf.get('api', {}).get('prefix', '/api/v2/')
+    filesize = None
+    key = filepath
+    status = 'in_progress'
+    if filepath.startswith('ftp://') or uri:
+      status = 'completed'
+    else:
+      key = 'uploads/{}/{}/{}'.format(
+        self.conf['user']['id'],
+        sample_id,
+        os.path.basename(filepath)
+      )
+      filesize = os.stat(filepath).st_size if os.path.exists(filepath) else 0
+
+    # update record with location of uploaded file
+    info = (Upload(self.conf.get('api'))).save(payload={
+        'filesize': filesize,
+        'is_paired_end': is_paired_end,
+        'key': key,
+        'order': order,
+        'sample': '{}samples/{}'.format(prefix, sample_id),
+        'source': source,
+        'status': status,
+        'uri': uri,
+        # 'timetaken': seconds_to_upload,
+    })
+    upload_id = info.get('id')
+    if upload_id:
+      if self.verbose:
+        eprint('\tcreated: upload with id', upload_id)
+      return upload_id, filepath, key
+    return None, None, None
+
+  def delete_upload(self, uid):
+    '''Delete method'''
+    return (Upload(self.conf.get('api'))).delete(uid)
+
+  def get_upload(self, uid):
+    '''Get resource'''
+    return (Upload(self.conf.get('api'))).get(
+      uid,
+      cache='{}/json/upload.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  def get_uploads(self):
+    '''Get resource list'''
+    info = (Upload(self.conf.get('api'))).list(
+        params={'limit': 0}
+    )
+    return info.get('objects', [])
+
+  def update_upload(self, uid, data):
+    '''Update resource'''
+    return (Upload(self.conf.get('api'))).save(obj_id=uid, payload=data)
+
+  def upload_file(self, upload_id, filepath, key):
+    '''Upload file to S3 and update info'''
+    starttime = time.time()
+    self.copy_file(filepath, key, action='to')
+    seconds_to_upload = int(time.time() - starttime)
+    (Upload(self.conf.get('api'))).save(obj_id=upload_id, payload={
+        'filesize': os.stat(filepath).st_size,
+        'seq_length': 0,
+        'status': 'completed',
+        'timetaken': seconds_to_upload,
+    })
+
+  def upload_uri_to_id(self, uri):
+    '''Get upload from uri and return upload id'''
+    response = (Upload(self.conf.get('api'))).list({'limit': 2, 'uri': uri})
+    items = response.get('objects')
+    if not items:
+      eprint('WARNING: no upload by uri', uri)
+      return None
+    if len(items) > 1:
+      eprint('WARNING: multiple upload by uri', uri)
+    return items[0].get('id')
+
+  ################################################################################################
+  ### USER #######################################################################################
+  ################################################################################################
+  def get_user(self, uid):
+    '''Get resource'''
+    return (User(self.conf.get('api'))).get(
+        uid,
+        cache='{}/json/user.{}.json'.format(self.scratch, uid) if self.use_cache else False,
+    )
+
+  ################################################################################################
+  ### GENERAL HELPERS AND METHODS ################################################################
+  ################################################################################################
+  def copy_file(self, src, dest, action='to'):
+    '''Copy file'''
+    return self.copy_file_to_s3(src, dest) if action == 'to' \
+        else self.copy_file_from_s3(src, dest)
+
+  def copy_file_from_s3(self, src, dest):
+    '''
+    Low level function to copy a file from cloud to disk
+    Parameters
+    ----------
+    src:  {str} File path on AWS S3, not including the bucket           [Required]
+    dest: {str} Directory or file path where file will be downloaded to [Required]
+    '''
+    storage_cfg = self.configuration.get_user_storage()
+    if not src.startswith('s3://'):
+      src = 's3://{}/{}'.format(storage_cfg.get('bucket'), src)
+    cmd = self.get_copy_cmd(src, dest)
+    try:
+      return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as stderr:
+      eprint("Error downloading {}.".format(src))
+      eprint("Return code: {}".format(stderr.returncode))
+      eprint("Ouput: {}".format(stderr.output))
+      return False
+
+  def copy_file_to_s3(self, src, dest, params=None):
+    '''Low level function to copy a file to cloud from disk'''
+    storage_cfg = self.configuration.get_user_storage()
+    dest = 's3://{}/{}'.format(storage_cfg.get('bucket'), dest)
+    cmd = self.get_copy_cmd(src, dest, sse=True, params=params)
+    if self.verbose:
+      eprint('copying', src, 'to', dest)
+    try:
+      return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+    except CalledProcessError as error:
+      eprint('error in copy file to s3')
+      eprint(cmd)
+      eprint(error.output)
+      return None
+
+  def download_file(self, filekey, dirname=None, filename=None, is_json=False, load=False): # pylint: disable=too-many-arguments
+    '''
+    High level function, downloads to scratch dir and opens and
+    parses files to JSON if asked. Uses low level copy_file()
+    Parameters
+    ----------
+    filekey:  {str}   The full file path on AWS S3               [Required]
+    dirname:  {str}   Directory to download the file to
+    filename: {str}   Give the download file a new name
+    isjson:   {bool}  Whether the downloaded file is JSON format
+    load:     {bool}  Load the file after downloading
+    Returns
+    -------
+    Absolute filepath to downloaded file
+    '''
+
+    if not filename:
+      # don't renaming the file
+      # get the download directory
+      prefix = self.scratch
+      if dirname:
+        prefix = dirname if dirname.startswith('/') else os.path.join(self.scratch, dirname)
+      filepath = os.path.join(prefix, os.path.basename(filekey))
+    else:
+      # rename the download file
+      filepath = self.get_filepath(filename, dirname=dirname)
+    filepath = os.path.expanduser(filepath)
+
+    # if file not already there, download it
+    if not os.path.exists(filepath) or os.path.getsize(filepath) == 0:
+      if self.verbose:
+        eprint('downloading', filepath)
+      if not os.path.exists(os.path.dirname(filepath)):
+        os.makedirs(os.path.dirname(filepath))
+      self.copy_file(filekey, filepath, action='from')
+    elif self.verbose:
+      eprint('exists', filepath)
+
+    if load:
+      data = open(filepath, 'r').read().strip()
+      return json.loads(data) if is_json else data
+    return filepath
+
+  def download_raw_files(self, sample, outdir=None):
+    '''
+    Download raw data associated with a sample
+    Parameters
+    ----------
+    sample: {dict} From calling bp.get_sample()       [Required]
+    outdir: {str}  Output directory to save files to
+    '''
+    uploads = sample['uploads']
+    files = [(upload.get('uri') or upload.get('key')) for upload in uploads]
+    files_downloaded = []
+    for file_i in files:
+      outdir = outdir if outdir else self.scratch
+      fileout = os.path.join(outdir, os.path.split(file_i)[1])
+      if os.path.exists(fileout):
+        eprint('Not downloading. File exists: {}'.format(fileout))
+        output = True
+      else:
+        output = self.copy_file_from_s3(file_i, outdir)
+
+      if output:
+        eprint('Downloaded {} to {}.'.format(os.path.split(file_i)[1], outdir))
+        files_downloaded.append(fileout)
+    return files_downloaded
+
+  @classmethod
+  def filter_files_by_tags(cls, files, tags, exclude=None, kind='exact', multiple=False): # pylint: disable=too-many-arguments
+    '''
+    Filter files that have all the tags. If exclude tags provided,
+    only return the files that don't have the exclude tags.append
+    In some cases, multiple files are expected, set it to true.
+    Parameters
+    ---------
+    files:    {dict}  List dictionaries containing file information     [Required]
+    tags:     {list}  List of tags                                      [Required]
+    kind:     {str}   Type of tag filtering. Options: exact, subset
+    exclude:  {list}  List of tags to exclude
+    multiple: {bool}  Whether to return all matching files or only one
+    Returns
+    -------
+    A list of files
+    '''
+
+    if tags is None:
+      return files
+
+    if not isinstance(tags, list):
+      eprint('Invalid tags argument. Provide a list of tags.')
+      return None
+
+    # filter for matches
+    files = [
+      file for file in files if file['tags'] and getattr(SetFilter, kind)(tags, file['tags'])
+    ]
+
+    # exclude files by tag
+    if exclude:
+      files = [
+        file for file in files if file['tags'] and SetFilter.diff(file['tags'], exclude)
+      ]
+
+    # check if no files left after filtering
+    if len(files) == 0:
+      eprint('No matching files after filtering by tags:', tags)
+      return None
+
+    # return files
+    if multiple:
+      return files
+    if len(files) > 1:
+      eprint('multiple matches for tags:', tags)
+      return files[0:1]
+    return files
+
+  def get_analysis_files(self, analysis, dirname=None, kind='exact', tags=None):  # pylint: disable=too-many-arguments
+    '''
+    For a analysis, go through files and get that match the tags
+    Parameters
+    ----------
+    analysis: {dict}  The dictionary returned by bp.get_analysis()  [Required]
+    dirname:  {str}   Directory to download files to
+    kind:     {str}   Type of tag filtering. Options: exact, subset
+    tags:     {list}  List of lists of tags to filter by
+    '''
+    # some input checking
+    if tags:
+      is_not_valid = not (isinstance(tags, list) and all([isinstance(item, list) for item in tags]))
+      if is_not_valid:
+        eprint('Invald tags argument. Provide a list of list of tags.')
+        return None
+    else:
+      tags = [None]
+
+    # filter the files
+    matching_files = []
+    for tags_sub in tags:
+      matching_files += self.filter_files_by_tags(
+        analysis['files'],
+        tags_sub,
+        kind=kind,
+        multiple=True,
+      )
+
+    return [self.download_file(
+      matching_file['path'],
+      dirname=dirname,
+    ) for matching_file in matching_files] if matching_files else None
+
+  def get_bam_file(self, sample, tags=None, multiple=False):
+    '''Get bam file. If you want deduped bam file, call with tags = ['bam', 'dedup']'''
+    return self.get_file_by_tags(
+      sample,
+      analysis_tags=['alignment'],
+      multiple=multiple,
+      tags=tags if tags else ['bam'],
+    )
+
+  def get_bigwig_file(self, sample, multiple=False):
+    '''Get bigwig file - for ChIP-Seq'''
+    return self.get_file_by_tags(
+      sample,
+      analysis_tags=['alignment'],
+      kind='exact',
+      multiple=multiple,
+      tags=['bigwig'],
+    )
+
+  def get_copy_cmd(self, src, dest, sse=False, params=None):
+    '''
+    Create copy cmd based on AWS cli utility
+    Parameters
+    ----------
+    src:    {str}     Path to file on AWS S3                                          [Required]
+    dest:   {str}     Destination directory or path where file will be downloaded to  [Required]
+    sse:    {bool}    Whether to use sever side encryption
+    params: {dict}    Other download paramters
+    '''
+    _params = ''
+    if sse:
+      _params = '--sse'
+    if params:
+      for arg, val in params.items():
+        _params += ' {} "{}"'.format(arg, val)
+
+    storage_cfg = self.configuration.get_user_storage()
+    credential = self.configuration.get_cli_credentials_from(storage_cfg)
+    return '{}aws s3 cp "{}" "{}" {}'.format(credential, src, dest, _params)
+
+  def get_expression_count_file(self, sample, features='transcripts', multiple=False):
+    '''Get expression count text file - for RNA-Seq'''
+    tags = ['expression_count', 'by_transcript', 'text']
+    if features == 'genes':
+      tags = ['expression_count', 'by_gene', 'text']
+    if self.verbose:
+      eprint('getting file w tags', tags)
+    return self.get_file_by_tags(
+      sample,
+      analysis_tags=['alignment'],
+      kind='exact',
+      multiple=multiple,
+      tags=tags,
+    )
+
+  def get_file_by_node(
+    self,
+    sample,
+    analysis_tags=None,
+    dest=None,
+    dirname=None,
+    multiple=False,
+    node=None,
+  ): # pylint: disable=too-many-arguments,too-many-branches,too-many-locals
+    '''For a sample, go through analysis and get files that match the tags'''
+    matches = []
+    for analysis in sample['analyses_full']:
+      should_continue = analysis['status'] == 'error' or (analysis_tags and not (
+        analysis['tags'] and (set(analysis_tags) <= set(analysis['tags']))
+      ))
+      if should_continue:
+        continue
+
+      matching_file = self._filter_files_by_node(
+          analysis['files'], node, multiple=multiple)
+
+      if matching_file:
+        if multiple:
+          for file in matching_file:
+            matches.append([analysis['id'], file])
+        else:
+          matches.append([analysis['id'], matching_file])
+
+    if len(matches) == 0:
+      eprint('WARNING: no matching file for', node)
+      return None
+
+    if len(matches) > 1:
+      matches.sort(
+        key=lambda item: datetime.datetime.strptime(item[1]['last_updated'], '%Y-%m-%dT%H:%M:%S.%f'),
+        reverse=True,
+      )
+      if not multiple:
+        eprint('WARNING: multiple matching file for', node)
+      for file in matches:
+        eprint('\t', file[1]['last_updated'], file[1]['path'])
+
+    if multiple:
+      filepath = []
+      for match in matches:
+        file1 = match[1]
+        path = self.download_file(file1['path'], dirname=dirname)
+        filepath.append(path)
+    else:
+      file1 = matches[0][1]
+      if dest:
+        dirname = dest
+      filepath = self.download_file(file1['path'], filename=dest, dirname=dirname)
+    return filepath
+
+  def get_file_by_tags(
+    self,
+    sample,
+    analysis_tags=None,
+    dest=None,
+    dirname=None,
+    download=True,
+    exclude=None,
+    kind='exact',
+    multiple=False,
+    tags=None,
+    #workflow_id=None,
+  ): # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements
+    '''
+    For a sample, go through analysis and get files that match the tags.
+    Parameters
+    ----------
+    sample:        {dict} Sample information                                       [Required]
+    analysis_tags: {list} Analysis tags to filter analyses when looking for files
+    dest:          {str}  File name to save the file to
+    dirname:       {str}  Ouput directory to save the file to
+    download:      {bool} Whether to download the file(s)
+    exclude:       {list} List of tags to exclude
+    kind:          {str}  Type of tag filtering to do. Options: exact, diff, or subset
+    multiple:      {bool} Whether to return multiple files or just the first one
+    tags:          {list} List of list of tags for file filtering. If just list of tags, will convert to list of lists.
+    workflow_id:   {int}  Workflow id to look for files in. NOT IMPLEMENTED YE
+    '''
+    # some error checking
+    if tags is None:
+      eprint("Tags is None")
+      return None
+
+    # make sure tags is a list of lists
+    if isinstance(tags, list):
+      if not isinstance(tags[0], list):
+        tags = [tags]
+
+    matches = []
+    matching_file = []
+    for analysis in sample['analyses_full']:
+      if analysis['status'] == 'error':
+        eprint('analysis ended in error, skipping.')
+        continue
+
+      # dont even look if not the right type of analysis
+      should_continue = analysis_tags and not(
+        analysis['tags'] and set(analysis_tags) <= set(analysis['tags'])
+      )
+      if should_continue:
+        continue
+
+      if analysis['params'] and 'info' in analysis['params']:
+        if not analysis['params']['info'].get('genome_id'):
+          eprint('Could not find genome for analysis {}'.format(analysis['id']))
+          continue
+        sample_genome_id = self.get_id_from_url(sample.get('genome', ''))
+        if not sample_genome_id:
+          eprint('Could not find genome for analysis {}'.format(analysis['id']))
+          continue
+        if int(analysis['params']['info']['genome_id']) != int(sample_genome_id):
+          eprint(
+            'analysis genome {}'.format(analysis['params']['info']['genome_id']),
+            'different from sample genome {}.'.format(sample_genome_id),
+          )
+          continue
+
+      if self.verbose:
+        eprint('looking at', analysis['id'], 'status', analysis['status'])
+      for tags_sub in tags:
+        filtered_files = self.filter_files_by_tags(
+          analysis['files'],
+          tags_sub,
+          exclude=exclude,
+          kind=kind,
+          multiple=multiple,
+        )
+
+        if filtered_files:
+          matching_file += filtered_files
+
+      if matching_file:
+        if multiple:
+          for file in matching_file:
+            matches.append([analysis['id'], file])
+        else:
+          matches.append([analysis['id'], matching_file[0]])
+
+    if not matches:
+      eprint('WARNING: no matching file for', tags)
+      eprint('in analyses with ids', [analysis['id'] for analysis in sample['analyses_full']])
+      eprint('for sample', sample['id'])
+      return None
+
+    if len(matches) > 1:
+      matches.sort(
+        key=lambda item: datetime.datetime.strptime(item[1]['last_updated'], '%Y-%m-%dT%H:%M:%S.%f'),
+        reverse=True,
+      )
+      if not multiple:
+        eprint('WARNING: multiple matching file for', tags)
+      for file in matches:
+        eprint('\t', file[1]['last_updated'], file[1]['path'])
+
+    if multiple:
+      filepath = []
+      for match in matches:
+        file1 = match[1]
+
+        if download:
+          path = self.download_file(file1['path'], dirname=dirname)
+          filepath.append(path)
+        else:
+          filepath.append(file1['path'])
+    else:
+      file1 = matches[0][1]
+      if download:
+        filepath = self.download_file(file1['path'], filename=dest, dirname=dirname)
+      else:
+        filepath = file1['path']
+        eprint('else')
+    return filepath
+
+  def get_filepath(self, filename, dirname=None):
+    '''Use scratch and [dirname] to construct a full filepath'''
+    # move along, nothin to do
+    if filename.startswith('/'):
+      filepath = filename
+    else:
+      if dirname:
+        filename = '{}/{}'.format(dirname.rstrip('/'), filename)
+      # adding dirname made it full
+      filepath = filename if filename.startswith('/') else self.scratch + '/' + filename
+    return filepath
+
+  def get_history(self, analysis_id, outdir=None):
+    '''
+    Get SWF history for the analysis
+    Parameters
+    ----------
+    analysis_id: {int} Analysis ID                                      [Required]
+    outdir:      {str} Directory to save data. Use instead of scratch.
+    '''
+    user_id = self._get_analysis_owner_id(analysis_id)
+    filekey = 'log/analyses/{}/{}/history.json'.format(user_id, analysis_id)
+    filename = '{}/history/history.{}.json'.format(
+      outdir if outdir else self.scratch,
+      analysis_id
+    )
+    return self.download_file(filekey, filename, load=True, is_json=True)
+
+  def get_id_from_url(self, url):
+    '''Parse URL to get the id'''
+    return self.parse_url(url)['id']
+
+  def get_log(self, analysis_id, outdir=None):
+    '''
+    Get log file for the analysis
+    Parameters
+    ----------
+    analysis_id: {int} Analysis ID                                      [Required]
+    outdir: {str}      Directory to save data. Use instead of scratch.
+    '''
+    user_id = self._get_analysis_owner_id(analysis_id)
+    filekey = 'log/analyses/{}/{}/worker.log'.format(user_id, analysis_id)
+    filename = '{}/worker.{}.log'.format(outdir if outdir else self.scratch, analysis_id)
+    return self.download_file(filekey, filename, load=False, is_json=False)
+
+  def get_window_score_filename(self, sample, kind=None, flanking=None):
+    '''Get window score filename'''
+    suffix = 'score-{}-{}Kb'.format(kind, flanking / 1e3)
+    suffix = suffix.replace('.0Kb', 'Kb')
+    return '{}/score/{}.{}.dedup.{}.txt'.format(
+      self.scratch,
+      sample['slug'],
+      sample['genome'],
+      suffix,
+    )
+
+  @classmethod
+  def parse_url(cls, url):
+    '''Parse URL to get the id, and other stuff'''
+    parts = url.rsplit('/', 1)
+    return {'id': parts[1]}
+
+  def print_data(self, data_type='', is_json=False, uid=None):
+    '''
+    Print data associated with genomes, samples, etc..
+    Parameters
+    ----------
+    data_type: {str}   Type of data to print (e.g. workflows)
+    is_json:      {bool}  By default, data is printed in a human-readable format
+    uid:       {list}  One or more ids of the objects you want
+    '''
+
+    if not isinstance(uid, list):
+      uid = [uid]
+
+    detail_methods = {
+      'analysis': 'get_analysis',
+      'genome': 'get_genome',
+      'sample': 'get_sample',
+    }
+
+    list_methods = {
+      'analyses': 'get_analyses',
+      'genomes': 'get_genomes',
+      'samples': 'get_samples',
+      'workflows': 'get_workflows',
+    }
+
+    # get the appropriate data
+    data = []
+
+    # if it is a detail
+    if data_type in detail_methods:
+      if uid is None:
+        eprint('Your uid is invalid: {}'.format(uid))
+        return
+
+      for item_id in uid:
+        data_tmp = getattr(self, list_methods.get(data_type))(item_id)
+        if data_tmp.get('id'):
+          data.append(data_tmp)
+        else:
+          eprint('{} with uid {} invalid.'.format(data_type.capitalize(), item_id))
+
+    # if it is a list
+    if data_type in list_methods:
+      data = getattr(self, list_methods.get(data_type))()
+
+
+    if not data:
+      eprint('Nothing found for the parameters you gave.')
+      return
+
+    # print the data as json
+    if is_json:
+      for item in data:
+        eprint(item)
+        eprint()
+      return
+
+    # print the data human readable
+    getattr(NicePrint, data_type)(data)
+
+  ### Private methods ###
+  def _add_full_analysis(self, sample):
+    '''Add full analysis info to the sample'''
+    analysis_ids = [self.parse_url(uri)['id'] for uri in sample['analyses']]
+    analyses = [self.get_analysis(uid) for uid in analysis_ids]
+    # remove null analyses, probably deleted or no ownership
+    analyses = [analysis for analysis in analyses if not analysis.get('error')]
+    sample['analyses_full'] = analyses
+    return sample
+
+  def _check_sample(self, uid):
+    '''Check if the sample is in the Basepair database'''
+    info = self.get_sample(uid, add_analysis=False)
+    if info.get('id'):
+      return True
+    eprint('The provided sample id: {id}, does not exist in Basepair.'.format(id=uid))
+    return False
+
+  def _check_workflow(self, uid):
+    '''Check if the workflow is in the Basepair database'''
+    info = self.get_workflow(uid)
+    if info.get('id'):
+      return True
+    eprint('The provided workflow id: {id}, does not exist in Basepair.'.format(id=uid))
+    return False
+
+  @classmethod
+  def _filter_files_by_node(cls, files, node, multiple=False):
+    '''Filter files that are from the node.'''
+    # In some cases, multiple files are expected, set it to true.
+    files = [file for file in files if file['node'] == node]
+    if multiple:
+      return files
+    if not files:
+      return None
+    if len(files) > 1:
+      eprint('multiple matches for node:', node)
+    return files[0]
+
+  def _get_analysis_owner_id(self, analysis_id):
+    '''Get analysis owner id'''
+    info = self.get_analysis(analysis_id)
+    return self.parse_url(info['owner'])['id'] if info else None
+
+  def _get_genome_by_name(self, genome_name):
+    '''Check if the genome is in the Basepair database'''
+    if genome_name:
+      available_genomes = [item for item in self.genomes if item.get('name') == genome_name]
+      if not available_genomes:
+        eprint(
+          'The provided genome, {}, does not exist in Basepair. Proceeding anyway...'.format(genome_name)
+        )
+        return None
+      return available_genomes[0].get('resource_uri')
+    return None
+
+  def _get_sample_owner_id(self, sample_id):
+    '''Get sample owner id'''
+    info = self.get_sample(sample_id)
+    return self.parse_url(info['owner'])['id'] if info else None
+
+  def _get_user_id(self):
+    '''Get userid, username provided in conf'''
+    user = (User(self.conf.get('api'))).list(
+      params={'limit': 1, 'username': self.conf['api']['username']}
+    )
+    self.conf['user'] = user.get('objects', [None])[0]
+
+  @classmethod
+  def _parsed_sample_list(cls, items, prefix):
+    '''Parse sample id list into sample resurce uri list'''
+    return ['{}samples/{}'.format(prefix, item_id) for item_id in items]