capsul 2.3.1__tar.gz → 2.3.2__tar.gz

{capsul-2.3.1/capsul.egg-info → capsul-2.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: capsul
-Version: 2.3.1
+Version: 2.3.2
 Summary: CAPSUL
 Home-page: https://github.com/populse/capsul
 Author: Populse team
@@ -14,10 +14,24 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Utilities
+License-File: LICENSE
+Requires-Dist: traits>=4.0
+Requires-Dist: soma-base>=4.6.1
+Requires-Dist: soma-workflow>=2.9.0
+Requires-Dist: populse-db>=1.1.1
+Requires-Dist: six>=1.13
+Requires-Dist: PyYAML
 Provides-Extra: doc
+Requires-Dist: sphinx>=1.0; extra == "doc"
+Requires-Dist: numpy>=1.3; extra == "doc"
 Provides-Extra: database
+Requires-Dist: populse_db; extra == "database"
 Provides-Extra: nipype
-License-File: LICENSE
+Requires-Dist: numpy>=1.3; extra == "nipype"
+Requires-Dist: scipy>=0.7; extra == "nipype"
+Requires-Dist: nibabel>=1.0; extra == "nipype"
+Requires-Dist: networkx>=1.0; extra == "nipype"
+Requires-Dist: nipype==0.10.0; extra == "nipype"
 
 
 ========

{capsul-2.3.1 → capsul-2.3.2}/capsul/attributes/completion_engine_iteration.py

@@ -247,12 +247,19 @@ class ProcessCompletionEngineIteration(ProcessCompletionEngine):
                 value = getattr(process.process, parameter)
                 iterative_parameters[parameter].append(value)
             self.completion_progress = it_step + 1
-        for parameter, values in six.iteritems(iterative_parameters):
+        for parameter, values in iterative_parameters.items():
             try:
                 setattr(process, parameter, values)
             except Exception as e:
                 print('assign iteration parameter', parameter, ':\n', e,
                       file=sys.stderr)
+        for parameter in parameters:
+            try:
+                value = getattr(process.process, parameter)
+                setattr(process, parameter, value)
+            except Exception as e:
+                print('assign parameter', parameter, ':\n', e,
+                      file=sys.stderr)
 
 
     def complete_iteration_step(self, step):

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/database_populse.py

@@ -14,7 +14,7 @@ class PopulseDBEngine(DatabaseEngine):
         self.db = Database(database_engine)
         with self.db as dbs:
             if not dbs.get_collection('path_metadata'):
-                # Create the schema if it does not exists                
+                # Create the schema if it does not exist
                 dbs.add_collection('named_directory', 'name')
                 dbs.add_field('named_directory', 'path', 'string')
                 

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/afni.py

@@ -22,6 +22,22 @@ def init_settings(capsul_engine):
                                      'afni'})
 
 
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('directory', ):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    return invalid
+
+
 def activate_configurations():
     '''
     Activate the AFNI module (set env variables) from the global configurations,

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/ants.py

@@ -22,6 +22,22 @@ def init_settings(capsul_engine):
                                      'ants'})
 
 
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('directory', ):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    return invalid
+
+
 def activate_configurations():
     '''
     Activate the ANTS module (set env variables) from the global configurations,

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/axon.py

@@ -59,6 +59,23 @@ def check_configurations():
         return 'Axon shared_directory is not found'
     return None
 
+
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('shared_directory', ):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    return invalid
+
+
 def complete_configurations():
     '''
     Try to automatically set or complete the capsul.engine.configurations for

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/fom.py

@@ -88,7 +88,9 @@ def init_settings(capsul_engine):
 
 
 def config_dependencies(config):
-    return {'axon': 'any', 'spm': 'any', 'attributes': 'any'}
+    return {'axon': 'any',
+            #'spm': 'any',
+            'attributes': 'any'}
 
 
 def config_updated(capsul_engine, environment, param=None, value=None):

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/freesurfer.py

@@ -46,6 +46,22 @@ def check_configurations():
     return None
 
 
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('setup', 'subjects_dir'):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    return invalid
+
+
 def complete_configurations():
     '''
     Try to automatically set or complete the capsul.engine.configurations for

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/fsl.py

@@ -47,12 +47,28 @@ def check_configurations():
         if fsl_prefix:
             return 'FSL configuration must either use config or prefix but not both'
         if not osp.exists(fsl_config):
-            return 'File "%s" does not exists' % fsl_config
+            return 'File "%s" does not exist' % fsl_config
         if not fsl_config.endswith('fsl.sh'):
             return 'File "%s" is not a path to fsl.sh script' % fsl_config
     return None
 
 
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('directory', 'config'):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    return invalid
+
+
 def complete_configurations():
     '''
     Try to automatically set or complete the capsul.engine.configurations for FSL.

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/matlab.py

@@ -28,13 +28,37 @@ def check_configurations():
     '''
     matlab_executable = capsul.engine.configurations.get(
         'capsul.engine.module.matlab', {}).get('executable')
-    if not matlab_executable:
-        return 'matlab.executable is not defined'
-    if not os.path.exists(matlab_executable):
+    mcr = capsul.engine.configurations.get(
+        'capsul.engine.module.matlab', {}).get('mcr_directory')
+    if not matlab_executable and not mcr:
+        return \
+            'both matlab.executable and matlab.mcr_directory are not defined'
+    if matlab_executable and not os.path.exists(matlab_executable):
         return 'Matlab executable is defined as "%s" but this path does not exist' % matlab_executable
     return None
 
 
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('executable', 'mcr_directory'):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    if len(invalid) == 1:
+        # if one of the paths is filled, then it should be OK.
+        # however it may depend on uses (SPM standalone needs mcr_directory,
+        # for instance)
+        return []
+    return invalid
+
+
 def edition_widget(engine, environment, config_id='matlab'):
     ''' Edition GUI for matlab config - see
     :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor`
@@ -62,22 +86,24 @@ def edition_widget(engine, environment, config_id='matlab'):
                 session.new_config(config_id, widget.environment, values)
             else:
                 for k in ('executable', 'mcr_directory', ):
-                    if (k == 'mcr_directory' and
-                                   values[k] and
-                                   not os.path.isdir(values[k])):
-                        raise NotADirectoryError('\nMatlab mcr_directory '
-                                                 'was not updated:\n{} is '
-                                                 'not existing!'.format(
-                                                                values[k]))
-                    elif (k == 'executable' and
-                                  values[k] and
-                                  not os.path.isfile(values[k])):
-                        raise FileNotFoundError('\nMatlab executable '
-                                                 'was not updated:\n{} is '
-                                                 'not existing!'.format(
-                                                                values[k]))
-                    else:
-                        setattr(conf, k, values[k])
+                    # don't check invalid files: they may be valid on a remote
+                    # server
+                    #if (k == 'mcr_directory' and
+                                   #values[k] and
+                                   #not os.path.isdir(values[k])):
+                        #raise NotADirectoryError('\nMatlab mcr_directory '
+                                                 #'was not updated:\n{} is '
+                                                 #'not existing!'.format(
+                                                                #values[k]))
+                    #elif (k == 'executable' and
+                                  #values[k] and
+                                  #not os.path.isfile(values[k])):
+                        #raise FileNotFoundError('\nMatlab executable '
+                                                 #'was not updated:\n{} is '
+                                                 #'not existing!'.format(
+                                                                #values[k]))
+                    #else:
+                    setattr(conf, k, values[k])
 
     controller = Controller()
     controller.add_trait('executable',

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/python.py

@@ -45,6 +45,22 @@ def activate_configurations():
         if py_path:
             sys.path = py_path + [p for p in sys.path if p not in py_path]
 
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('executable', ):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    return invalid
+
+
 def edition_widget(engine, environment, config_id='python'):
     ''' Edition GUI for python config - see
     :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor`

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/module/spm.py

@@ -33,8 +33,7 @@ def init_settings(capsul_engine):
 
 
 def config_dependencies(config):
-        if not config['standalone']:
-            return {'matlab': 'any'}
+    return {'matlab': 'any'}
 
 #def set_environ(config, environ):
     #spm_config = config.get('spm', {})
@@ -160,6 +159,22 @@ def activate_configurations():
         del os.environ['SPM_STANDALONE']
 
 
+def check_notably_invalid_config(conf):
+    '''
+    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled
+
+    Returns
+    -------
+    invalid: list
+        list of invalid config keys
+    '''
+    invalid = []
+    for k in ('directory', 'version'):
+        if getattr(conf, k, None) is None:
+            invalid.append(k)
+    return invalid
+
+
 def edition_widget(engine, environment, config_id='any'):
     ''' Edition GUI for SPM config - see
     :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor`

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/settings.py

@@ -98,7 +98,8 @@ class Settings:
             module_name = 'capsul.engine.module.' + module_name
         return module_name
     
-    def select_configurations(self, environment, uses=None):
+    def select_configurations(self, environment, uses=None,
+                              check_invalid_mods=False):
         '''
         Select a configuration for a given environment. A configuration is
         a dictionary whose keys are module names and values are
@@ -117,6 +118,9 @@ class Settings:
         environment and, if no result is found, the `'global'` environment
         (the value defined in `Settings.global_environment`) is used.
         
+        If `check_invalid_mods` is True, then each selected config module is
+        checked for missing values and discarded if there are.
+
         example
         -------
         To select a SPM version greater than 8 for an environment called
@@ -162,6 +166,19 @@ class Settings:
                                                                full_query))
                 else:
                     docs = []
+
+                if check_invalid_mods:
+                    # filter out invalid configs
+                    doc_t = docs
+                    docs = []
+                    for doc in doc_t:
+                        mod = sys.modules[module]
+                        invalid = []
+                        if hasattr(mod, 'check_notably_invalid_config'):
+                            invalid = mod.check_notably_invalid_config(doc)
+                        if len(invalid) == 0:
+                            docs.append(doc)
+
                 if len(docs) == 1:
                     selected_config = docs[0]
                 elif len(docs) > 1:
@@ -183,6 +200,19 @@ class Settings:
                                                                    full_query))
                     else:
                         docs = []
+
+                    if check_invalid_mods:
+                        # filter out invalid configs
+                        doc_t = docs
+                        docs = []
+                        for doc in doc_t:
+                            mod = sys.modules[module]
+                            invalid = []
+                            if hasattr(mod, 'check_notably_invalid_config'):
+                                invalid = mod.check_notably_invalid_config(doc)
+                            if len(invalid) == 0:
+                                docs.append(doc)
+
                     if len(docs) == 1:
                         selected_config = docs[0]
                     elif len(docs) > 1:

{capsul-2.3.1 → capsul-2.3.2}/capsul/engine/test/test_capsul_engine.py

@@ -13,9 +13,8 @@ import json
 import glob
 
 from capsul.api import capsul_engine
-from capsul.api import Process, Pipeline
+from capsul.api import Process
 from capsul.engine import activate_configuration
-from capsul import engine
 from soma_workflow import configuration as swconfig
 from traits.api import File
 
@@ -63,13 +62,14 @@ class MatlabProcess(Process):
         if not mconf:
             raise RuntimeError('Matlab config is not present')
 
+
 class PythonProcess(Process):
     output_config = File(output=True, desc='output file to write config',
                          allowed_extensions=['.json'])
 
     # python requirements are handled automatically
-    #def requirements(self):
-        #return {'python': 'any'}
+    # def requirements(self):
+    #     return {'python': 'any'}
 
     def _run_process(self):
         import capsul.engine
@@ -91,13 +91,15 @@ def tearDownModule():
         del os.environ['HOME']
     else:
         os.environ['HOME'] = old_home
-    #print('temp_home_dir:', temp_home_dir)
+    # print('temp_home_dir:', temp_home_dir)
     shutil.rmtree(temp_home_dir)
 
 
 def check_nipype_spm():
     # look for hardcoded paths, I have no other way at hand...
     spm_standalone_paths = ['/usr/local/spm12-standalone',
+                            '/volatile/local/spm12-standalone',
+                            '/drf/local/spm12-standalone',
                             '/i2bm/local/spm12-standalone']
     spm_standalone_path = [p for p in spm_standalone_paths if os.path.isdir(p)]
     if not spm_standalone_path:
@@ -115,6 +117,9 @@ def check_nipype_spm():
                         mcr_path = l.split('=', 1)[1]
         if not mcr_path:
             mcr_paths = ['/usr/local/matlab/MATLAB_Runtime',
+                         '/volatile/local/spm12-standalone/mcr',
+                         '/volatile/local/MATLAB_Runtime',
+                         '/drf/local/matlab/MATLAB_Runtime',
                          '/i2bm/local/matlab/MATLAB_Runtime', ]
             for p in mcr_paths:
                 mcr = glob.glob(osp.join(p, 'v*'))
@@ -136,7 +141,7 @@ class TestCapsulEngine(unittest.TestCase):
     def setUp(self):
         self.sqlite_file = str(tempfile.mktemp(suffix='.sqlite'))
         self.ce = capsul_engine(self.sqlite_file)
-    
+
     def tearDown(self):
         self.ce = None
         # garbage collect to ensure the database is closed
@@ -146,29 +151,28 @@ class TestCapsulEngine(unittest.TestCase):
         if os.path.exists(self.sqlite_file):
             os.remove(self.sqlite_file)
 
-
     def test_engine_settings(self):
         # In the following, we use explicit values for config_id_field
         # (which is a single string value that must be unique for each
         # config). This is not mandatory but it avoids to have randomly
         # generated values making testing results more difficult to tackle.
         self.maxDiff = 2000
-        
+
         cif = self.ce.settings.config_id_field
         with self.ce.settings as settings:
             # Create a new section object for 'fsl' in 'global' environment
             config = settings.config('fsl', 'global')
             if config:
                 settings.remove_config('fsl', 'global',
-                                        getattr(config, cif))
-            fsl = settings.new_config('fsl', 'global', {cif:'5'})
+                                       getattr(config, cif))
+            fsl = settings.new_config('fsl', 'global', {cif: '5'})
             fsl.directory = '/there'
 
             # Create a global AFNI configuration
             config = settings.config('afni', 'global')
             if config:
                 settings.remove_config('afni', 'global',
-                                        getattr(config, cif))
+                                       getattr(config, cif))
             settings.new_config('afni', 'global', {'directory': '/there',
                                                    cif: '22'})
 
@@ -183,15 +187,15 @@ class TestCapsulEngine(unittest.TestCase):
             # Create two global SPM configurations
             settings.new_config('spm', 'global', {'version': '8',
                                                   'standalone': True,
-                                                  cif:'8'})
+                                                  cif: '8'})
             settings.new_config('spm', 'global', {'version': '12',
                                                   'standalone': False,
-                                                  cif:'12'})
+                                                  cif: '12'})
             # Create one SPM configuration for 'my_machine'
             settings.new_config('spm', 'my_machine', {'version': '20',
                                                       'standalone': True,
-                                                      cif:'20'})
-    
+                                                      cif: '20'})
+
         self.assertEqual(
             self.ce.settings.select_configurations('my_machine'),
             {'capsul_engine': {'uses': {'capsul.engine.module.fsl': 'ALL',
@@ -203,15 +207,15 @@ class TestCapsulEngine(unittest.TestCase):
                                           'directory': '/there',
                                           cif: '5'},
              'capsul.engine.module.afni': {
-                'config_environment': 'global','directory': '/there',
+                'config_environment': 'global', 'directory': '/there',
                 cif: '22'},
              'capsul.engine.module.ants': {
                  'config_environment': 'global', 'directory': '/there',
                  cif: '235'},
              'capsul.engine.module.spm': {'config_environment': 'my_machine',
-                                           'version': '20',
-                                           'standalone': True,
-                                           cif: '20'}})
+                                          'version': '20',
+                                          'standalone': True,
+                                          cif: '20'}})
         self.assertRaises(EnvironmentError,
                           lambda:
                               self.ce.settings.select_configurations('global'))
@@ -220,7 +224,7 @@ class TestCapsulEngine(unittest.TestCase):
                                                    uses={'fsl': 'any'}),
             {'capsul.engine.module.fsl':
                 {'config_environment': 'global', 'directory': '/there',
-                 cif:'5'},
+                 cif: '5'},
              'capsul_engine':
                 {'uses': {'capsul.engine.module.fsl': 'any'}}})
 
@@ -228,8 +232,8 @@ class TestCapsulEngine(unittest.TestCase):
             self.ce.settings.select_configurations('global',
                                                    uses={'afni': 'any'}),
             {'capsul.engine.module.afni':
-                 {'config_environment': 'global', 'directory': '/there',
-                  cif: '22'},
+                {'config_environment': 'global', 'directory': '/there',
+                 cif: '22'},
              'capsul_engine':
                  {'uses': {'capsul.engine.module.afni': 'any'}}})
 
@@ -237,8 +241,8 @@ class TestCapsulEngine(unittest.TestCase):
             self.ce.settings.select_configurations('global',
                                                    uses={'ants': 'any'}),
             {'capsul.engine.module.ants':
-                 {'config_environment': 'global', 'directory': '/there',
-                  cif: '235'},
+                {'config_environment': 'global', 'directory': '/there',
+                 cif: '235'},
              'capsul_engine':
                  {'uses': {'capsul.engine.module.ants': 'any'}}})
 
@@ -251,7 +255,8 @@ class TestCapsulEngine(unittest.TestCase):
                  'standalone': True,
                  cif: '8'},
              'capsul_engine':
-                {'uses': {'capsul.engine.module.spm': 'any'}}})
+                {'uses': {'capsul.engine.module.spm': 'any',
+                          'capsul.engine.module.matlab': 'any'}}})
         self.assertEqual(
             self.ce.settings.select_configurations(
                 'global',  uses={'spm': 'version=="12"'}),
@@ -292,7 +297,7 @@ print(sys.argv)
                 if config:
                     settings.remove_config('fsl', 'global',
                                            getattr(config, cif))
-                fsl = settings.new_config('fsl', 'global', {cif:'5'})
+                fsl = settings.new_config('fsl', 'global', {cif: '5'})
                 fsl.directory = tdir
                 fsl.prefix = 'fsl5.0-'
 
@@ -328,9 +333,7 @@ print(sys.argv)
     def test_nipype_spm_config(self):
         tdir = tempfile.mkdtemp(prefix='capsul_spm')
         try:
-            cif = self.ce.settings.config_id_field
             spm_path, mcr_path = check_nipype_spm()
-            # FIXME: do something with mcr_path
             t1_src = osp.join(spm_path,
                               'spm12_mcr/spm12/toolbox/OldNorm/T1.nii')
             if not osp.exists(t1_src):
@@ -347,10 +350,13 @@ print(sys.argv)
                                    {'directory': spm_path, 'standalone': True,
                                     'version': '12'})
                 session.new_config('nipype', 'global', {})
+                if mcr_path:
+                    session.new_config('matlab', 'global',
+                                       {'mcr_directory': mcr_path})
 
             self.ce.study_config.use_soma_workflow = True
             self.ce.study_config.somaworkflow_keep_failed_workflows = True
-            #self.ce.study_config.somaworkflow_keep_succeeded_workflows = True
+            # self.ce.study_config.somaworkflow_keep_succeeded_workflows = True
 
             conf = self.ce.settings.select_configurations(
                 'global', uses={'nipype': 'any', 'spm': 'any'})
@@ -364,7 +370,7 @@ print(sys.argv)
             self.assertTrue(osp.exists(osp.join(tdir, 'sT1.nii')))
 
         finally:
-            #print('tdir:', tdir)
+            # print('tdir:', tdir)
             shutil.rmtree(tdir)
 
     @unittest.skipIf(sys.platform.startswith('win'),
@@ -401,7 +407,7 @@ print(sys.argv)
                 {'config_id': 'matlab', 'executable': matlab_exe,
                  'config_environment': 'global'})
         finally:
-            #print('tdir:', tdir)
+            # print('tdir:', tdir)
             shutil.rmtree(tdir)
 
     @unittest.skipIf(sys.platform.startswith('win'),
@@ -446,7 +452,7 @@ print(sys.argv)
             for path in py_path:
                 self.assertTrue(path in paths)
         finally:
-            #print('tdir:', tdir)
+            # print('tdir:', tdir)
             shutil.rmtree(tdir)