# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
""" Set of interfaces that allow interaction with data. Currently
    available interfaces are:

    DataSource: Generic nifti to named Nifti interface
    DataSink: Generic named output from interfaces to data store
    XNATSource: preliminary interface to XNAT

    To come :
    XNATSink

    Change directory to provide relative paths for doctests
    >>> import os
    >>> filepath = os.path.dirname( os.path.realpath( __file__ ) )
    >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data'))
    >>> os.chdir(datadir)

"""
import glob
import fnmatch
import string
import os
import os.path as op
import shutil
import subprocess
import re
import tempfile
from warnings import warn

import sqlite3
from nipype.utils.misc import human_order_sorted
from nipype.external import six

try:
    import pyxnat
except:
    pass

try:
    import paramiko
except:
    pass

from nipype.interfaces.base import (TraitedSpec, traits, File, Directory,
                                    BaseInterface, InputMultiPath, isdefined,
                                    OutputMultiPath, DynamicTraitedSpec,
                                    Undefined, BaseInterfaceInputSpec)
from nipype.utils.filemanip import (copyfile, list_to_filename,
                                    filename_to_list)

from .. import logging
iflogger = logging.getLogger('interface')


def copytree(src, dst):
    """Recursively copy a directory tree using
    nipype.utils.filemanip.copyfile()

    This is not a thread-safe routine. However, in the case of creating new
    directories, it checks to see if a particular directory has already been
    created by another process.
    """
    names = os.listdir(src)
    try:
        os.makedirs(dst)
    except OSError, why:
        if 'File exists' in why:
            pass
        else:
            raise why
    errors = []
    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if os.path.isdir(srcname):
                copytree(srcname, dstname)
            else:
                copyfile(srcname, dstname, True, hashmethod='content')
        except (IOError, os.error), why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except Exception, err:
            errors.extend(err.args[0])
    if errors:
        raise Exception(errors)


def add_traits(base, names, trait_type=None):
    """ Add traits to a traited class.

    All traits are set to Undefined by default
    """
    if trait_type is None:
        trait_type = traits.Any
    undefined_traits = {}
    for key in names:
        base.add_trait(key, trait_type)
        undefined_traits[key] = Undefined
    base.trait_set(trait_change_notify=False, **undefined_traits)
    # access each trait
    for key in names:
        _ = getattr(base, key)
    return base


class IOBase(BaseInterface):

    def _run_interface(self, runtime):
        return runtime

    def _list_outputs(self):
        raise NotImplementedError

    def _outputs(self):
        return self._add_output_traits(super(IOBase, self)._outputs())

    def _add_output_traits(self, base):
        return base


class DataSinkInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
    base_directory = Directory(
        desc='Path to the base directory for storing data.')
    container = traits.Str(
        desc='Folder within base directory in which to store output')
    parameterization = traits.Bool(True, usedefault=True,
                                   desc='store output in parametrized structure')
    strip_dir = Directory(desc='path to strip out of filename')
    substitutions = InputMultiPath(traits.Tuple(traits.Str, traits.Str),
                                   desc=('List of 2-tuples reflecting string '
                                         'to substitute and string to replace '
                                         'it with'))
    regexp_substitutions = InputMultiPath(traits.Tuple(traits.Str, traits.Str),
                                          desc=('List of 2-tuples reflecting a pair '
                                                'of a Python regexp pattern and a '
                                                'replacement string. Invoked after '
                                                'string `substitutions`'))

    _outputs = traits.Dict(traits.Str, value={}, usedefault=True)
    remove_dest_dir = traits.Bool(False, usedefault=True,
                                  desc='remove dest directory when copying dirs')

    def __setattr__(self, key, value):
        if key not in self.copyable_trait_names():
            if not isdefined(value):
                super(DataSinkInputSpec, self).__setattr__(key, value)
            self._outputs[key] = value
        else:
            if key in self._outputs:
                self._outputs[key] = value
            super(DataSinkInputSpec, self).__setattr__(key, value)


class DataSinkOutputSpec(TraitedSpec):

    out_file = traits.Any(desc='datasink output')


class DataSink(IOBase):
    """ Generic datasink module to store structured outputs

        Primarily for use within a workflow. This interface allows arbitrary
        creation of input attributes. The names of these attributes define the
        directory structure to create for storage of the files or directories.

        The attributes take the following form:

        string[[.[@]]string[[.[@]]string]] ...

        where parts between [] are optional.

        An attribute such as contrasts.@con will create a 'contrasts' directory
        to store the results linked to the attribute. If the @ is left out, such
        as in 'contrasts.con', a subdirectory 'con' will be created under
        'contrasts'.

        the general form of the output is::

           'base_directory/container/parameterization/destloc/filename'

           destloc = string[[.[@]]string[[.[@]]string]] and
           filename comesfrom the input to the connect statement.

        .. warning::

            This is not a thread-safe node because it can write to a common
            shared location. It will not complain when it overwrites a file.

        .. note::

            If both substitutions and regexp_substitutions are used, then
            substitutions are applied first followed by regexp_substitutions.

            This interface **cannot** be used in a MapNode as the inputs are
            defined only when the connect statement is executed.

        Examples
        --------

        >>> ds = DataSink()
        >>> ds.inputs.base_directory = 'results_dir'
        >>> ds.inputs.container = 'subject'
        >>> ds.inputs.structural = 'structural.nii'
        >>> setattr(ds.inputs, 'contrasts.@con', ['cont1.nii', 'cont2.nii'])
        >>> setattr(ds.inputs, 'contrasts.alt', ['cont1a.nii', 'cont2a.nii'])
        >>> ds.run() # doctest: +SKIP

        To use DataSink in a MapNode, its inputs have to be defined at the
        time the interface is created.

        >>> ds = DataSink(infields=['contasts.@con'])
        >>> ds.inputs.base_directory = 'results_dir'
        >>> ds.inputs.container = 'subject'
        >>> ds.inputs.structural = 'structural.nii'
        >>> setattr(ds.inputs, 'contrasts.@con', ['cont1.nii', 'cont2.nii'])
        >>> setattr(ds.inputs, 'contrasts.alt', ['cont1a.nii', 'cont2a.nii'])
        >>> ds.run() # doctest: +SKIP

    """
    input_spec = DataSinkInputSpec
    output_spec = DataSinkOutputSpec

    def __init__(self, infields=None, force_run=True, **kwargs):
        """
        Parameters
        ----------
        infields : list of str
            Indicates the input fields to be dynamically created
        """

        super(DataSink, self).__init__(**kwargs)
        undefined_traits = {}
        # used for mandatory inputs check
        self._infields = infields
        if infields:
            for key in infields:
                self.inputs.add_trait(key, traits.Any)
                self.inputs._outputs[key] = Undefined
                undefined_traits[key] = Undefined
        self.inputs.trait_set(trait_change_notify=False, **undefined_traits)
        if force_run:
            self._always_run = True

    def _get_dst(self, src):
        ## If path is directory with trailing os.path.sep,
        ## then remove that for a more robust behavior
        src = src.rstrip(os.path.sep)
        path, fname = os.path.split(src)
        if self.inputs.parameterization:
            dst = path
            if isdefined(self.inputs.strip_dir):
                dst = dst.replace(self.inputs.strip_dir, '')
            folders = [folder for folder in dst.split(os.path.sep) if
                       folder.startswith('_')]
            dst = os.path.sep.join(folders)
            if fname:
                dst = os.path.join(dst, fname)
        else:
            if fname:
                dst = fname
            else:
                dst = path.split(os.path.sep)[-1]
        if dst[0] == os.path.sep:
            dst = dst[1:]
        return dst

    def _substitute(self, pathstr):
        pathstr_ = pathstr
        if isdefined(self.inputs.substitutions):
            for key, val in self.inputs.substitutions:
                oldpathstr = pathstr
                pathstr = pathstr.replace(key, val)
                if pathstr != oldpathstr:
                    iflogger.debug('sub.str: %s -> %s using %r -> %r'
                                   % (oldpathstr, pathstr, key, val))
        if isdefined(self.inputs.regexp_substitutions):
            for key, val in self.inputs.regexp_substitutions:
                oldpathstr = pathstr
                pathstr, _ = re.subn(key, val, pathstr)
                if pathstr != oldpathstr:
                    iflogger.debug('sub.regexp: %s -> %s using %r -> %r'
                                   % (oldpathstr, pathstr, key, val))
        if pathstr_ != pathstr:
            iflogger.info('sub: %s -> %s' % (pathstr_, pathstr))
        return pathstr

    def _list_outputs(self):
        """Execute this module.
        """
        outputs = self.output_spec().get()
        out_files = []
        outdir = self.inputs.base_directory
        if not isdefined(outdir):
            outdir = '.'
        outdir = os.path.abspath(outdir)
        if isdefined(self.inputs.container):
            outdir = os.path.join(outdir, self.inputs.container)
        if not os.path.exists(outdir):
            try:
                os.makedirs(outdir)
            except OSError, inst:
                if 'File exists' in inst:
                    pass
                else:
                    raise(inst)
        for key, files in self.inputs._outputs.items():
            if not isdefined(files):
                continue
            iflogger.debug("key: %s files: %s" % (key, str(files)))
            files = filename_to_list(files)
            tempoutdir = outdir
            for d in key.split('.'):
                if d[0] == '@':
                    continue
                tempoutdir = os.path.join(tempoutdir, d)

            # flattening list
            if isinstance(files, list):
                if isinstance(files[0], list):
                    files = [item for sublist in files for item in sublist]

            for src in filename_to_list(files):
                src = os.path.abspath(src)
                if os.path.isfile(src):
                    dst = self._get_dst(src)
                    dst = os.path.join(tempoutdir, dst)
                    dst = self._substitute(dst)
                    path, _ = os.path.split(dst)
                    if not os.path.exists(path):
                        try:
                            os.makedirs(path)
                        except OSError, inst:
                            if 'File exists' in inst:
                                pass
                            else:
                                raise(inst)
                    iflogger.debug("copyfile: %s %s" % (src, dst))
                    copyfile(src, dst, copy=True, hashmethod='content')
                    out_files.append(dst)
                elif os.path.isdir(src):
                    dst = self._get_dst(os.path.join(src, ''))
                    dst = os.path.join(tempoutdir, dst)
                    dst = self._substitute(dst)
                    path, _ = os.path.split(dst)
                    if not os.path.exists(path):
                        try:
                            os.makedirs(path)
                        except OSError, inst:
                            if 'File exists' in inst:
                                pass
                            else:
                                raise(inst)
                    if os.path.exists(dst) and self.inputs.remove_dest_dir:
                        iflogger.debug("removing: %s" % dst)
                        shutil.rmtree(dst)
                    iflogger.debug("copydir: %s %s" % (src, dst))
                    copytree(src, dst)
                    out_files.append(dst)
        outputs['out_file'] = out_files

        return outputs


class DataGrabberInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):  # InterfaceInputSpec):
    base_directory = Directory(exists=True,
                               desc='Path to the base directory consisting of subject data.')
    raise_on_empty = traits.Bool(True, usedefault=True,
                                 desc='Generate exception if list is empty for a given field')
    sort_filelist = traits.Bool(mandatory=True,
                                desc='Sort the filelist that matches the template')
    template = traits.Str(mandatory=True,
                          desc='Layout used to get files. relative to base directory if defined')
    template_args = traits.Dict(key_trait=traits.Str,
                                value_trait=traits.List(traits.List),
                                desc='Information to plug into template')


class DataGrabber(IOBase):
    """ Generic datagrabber module that wraps around glob in an
        intelligent way for neuroimaging tasks to grab files


        .. attention::

           Doesn't support directories currently

        Examples
        --------

        >>> from nipype.interfaces.io import DataGrabber

        Pick all files from current directory

        >>> dg = DataGrabber()
        >>> dg.inputs.template = '*'

        Pick file foo/foo.nii from current directory

        >>> dg.inputs.template = '%s/%s.dcm'
        >>> dg.inputs.template_args['outfiles']=[['dicomdir','123456-1-1.dcm']]

        Same thing but with dynamically created fields

        >>> dg = DataGrabber(infields=['arg1','arg2'])
        >>> dg.inputs.template = '%s/%s.nii'
        >>> dg.inputs.arg1 = 'foo'
        >>> dg.inputs.arg2 = 'foo'

        however this latter form can be used with iterables and iterfield in a
        pipeline.

        Dynamically created, user-defined input and output fields

        >>> dg = DataGrabber(infields=['sid'], outfields=['func','struct','ref'])
        >>> dg.inputs.base_directory = '.'
        >>> dg.inputs.template = '%s/%s.nii'
        >>> dg.inputs.template_args['func'] = [['sid',['f3','f5']]]
        >>> dg.inputs.template_args['struct'] = [['sid',['struct']]]
        >>> dg.inputs.template_args['ref'] = [['sid','ref']]
        >>> dg.inputs.sid = 's1'

        Change the template only for output field struct. The rest use the
        general template

        >>> dg.inputs.field_template = dict(struct='%s/struct.nii')
        >>> dg.inputs.template_args['struct'] = [['sid']]

    """
    input_spec = DataGrabberInputSpec
    output_spec = DynamicTraitedSpec
    _always_run = True

    def __init__(self, infields=None, outfields=None, **kwargs):
        """
        Parameters
        ----------
        infields : list of str
            Indicates the input fields to be dynamically created

        outfields: list of str
            Indicates output fields to be dynamically created

        See class examples for usage

        """
        if not outfields:
            outfields = ['outfiles']
        super(DataGrabber, self).__init__(**kwargs)
        undefined_traits = {}
        # used for mandatory inputs check
        self._infields = infields
        self._outfields = outfields
        if infields:
            for key in infields:
                self.inputs.add_trait(key, traits.Any)
                undefined_traits[key] = Undefined
        # add ability to insert field specific templates
        self.inputs.add_trait('field_template',
                              traits.Dict(traits.Enum(outfields),
                                          desc="arguments that fit into template"))
        undefined_traits['field_template'] = Undefined
        if not isdefined(self.inputs.template_args):
            self.inputs.template_args = {}
        for key in outfields:
            if not key in self.inputs.template_args:
                if infields:
                    self.inputs.template_args[key] = [infields]
                else:
                    self.inputs.template_args[key] = []

        self.inputs.trait_set(trait_change_notify=False, **undefined_traits)

    def _add_output_traits(self, base):
        """

        Using traits.Any instead out OutputMultiPath till add_trait bug
        is fixed.
        """
        return add_traits(base, self.inputs.template_args.keys())

    def _list_outputs(self):
        # infields are mandatory, however I could not figure out how to set 'mandatory' flag dynamically
        # hence manual check
        if self._infields:
            for key in self._infields:
                value = getattr(self.inputs, key)
                if not isdefined(value):
                    msg = "%s requires a value for input '%s' because it was listed in 'infields'" % \
                        (self.__class__.__name__, key)
                    raise ValueError(msg)

        outputs = {}
        for key, args in self.inputs.template_args.items():
            outputs[key] = []
            template = self.inputs.template
            if hasattr(self.inputs, 'field_template') and \
                    isdefined(self.inputs.field_template) and \
                    key in self.inputs.field_template:
                template = self.inputs.field_template[key]
            if isdefined(self.inputs.base_directory):
                template = os.path.join(
                    os.path.abspath(self.inputs.base_directory), template)
            else:
                template = os.path.abspath(template)
            if not args:
                filelist = glob.glob(template)
                if len(filelist) == 0:
                    msg = 'Output key: %s Template: %s returned no files' % (
                        key, template)
                    if self.inputs.raise_on_empty:
                        raise IOError(msg)
                    else:
                        warn(msg)
                else:
                    if self.inputs.sort_filelist:
                        filelist = human_order_sorted(filelist)
                    outputs[key] = list_to_filename(filelist)
            for argnum, arglist in enumerate(args):
                maxlen = 1
                for arg in arglist:
                    if isinstance(arg, six.string_types) and hasattr(self.inputs, arg):
                        arg = getattr(self.inputs, arg)
                    if isinstance(arg, list):
                        if (maxlen > 1) and (len(arg) != maxlen):
                            raise ValueError('incompatible number of arguments for %s' % key)
                        if len(arg) > maxlen:
                            maxlen = len(arg)
                outfiles = []
                for i in range(maxlen):
                    argtuple = []
                    for arg in arglist:
                        if isinstance(arg, six.string_types) and hasattr(self.inputs, arg):
                            arg = getattr(self.inputs, arg)
                        if isinstance(arg, list):
                            argtuple.append(arg[i])
                        else:
                            argtuple.append(arg)
                    filledtemplate = template
                    if argtuple:
                        try:
                            filledtemplate = template % tuple(argtuple)
                        except TypeError as e:
                            raise TypeError(e.message + ": Template %s failed to convert with args %s" % (template, str(tuple(argtuple))))
                    outfiles = glob.glob(filledtemplate)
                    if len(outfiles) == 0:
                        msg = 'Output key: %s Template: %s returned no files' % (key, filledtemplate)
                        if self.inputs.raise_on_empty:
                            raise IOError(msg)
                        else:
                            warn(msg)
                        outputs[key].append(None)
                    else:
                        if self.inputs.sort_filelist:
                            outfiles = human_order_sorted(outfiles)
                        outputs[key].append(list_to_filename(outfiles))
            if any([val is None for val in outputs[key]]):
                outputs[key] = []
            if len(outputs[key]) == 0:
                outputs[key] = None
            elif len(outputs[key]) == 1:
                outputs[key] = outputs[key][0]
        return outputs


class SelectFilesInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):

    base_directory = Directory(exists=True,
        desc="Root path common to templates.")
    sort_filelist = traits.Bool(True, usedefault=True,
        desc="When matching mutliple files, return them in sorted order.")
    raise_on_empty = traits.Bool(True, usedefault=True,
        desc="Raise an exception if a template pattern matches no files.")
    force_lists = traits.Either(traits.Bool(), traits.List(traits.Str()),
        default=False, usedefault=True,
        desc=("Whether to return outputs as a list even when only one file "
              "matches the template. Either a boolean that applies to all "
              "output fields or a list of output field names to coerce to "
              " a list"))


class SelectFiles(IOBase):
    """Flexibly collect data from disk to feed into workflows.

    This interface uses the {}-based string formatting syntax to plug
    values (possibly known only at workflow execution time) into string
    templates and collect files from persistant storage. These templates
    can also be combined with glob wildcards. The field names in the
    formatting template (i.e. the terms in braces) will become inputs
    fields on the interface, and the keys in the templates dictionary
    will form the output fields.

    Examples
    --------

    >>> from nipype import SelectFiles, Node
    >>> templates={"T1": "{subject_id}/struct/T1.nii",
    ...            "epi": "{subject_id}/func/f[0, 1].nii"}
    >>> dg = Node(SelectFiles(templates), "selectfiles")
    >>> dg.inputs.subject_id = "subj1"
    >>> dg.outputs.get()
    {'T1': <undefined>, 'epi': <undefined>}

    The same thing with dynamic grabbing of specific files:

    >>> templates["epi"] = "{subject_id}/func/f{run!s}.nii"
    >>> dg = Node(SelectFiles(templates), "selectfiles")
    >>> dg.inputs.subject_id = "subj1"
    >>> dg.inputs.run = [2, 4]

    """
    input_spec = SelectFilesInputSpec
    output_spec = DynamicTraitedSpec
    _always_run = True

    def __init__(self, templates, **kwargs):
        """Create an instance with specific input fields.

        Parameters
        ----------
        templates : dictionary
            Mapping from string keys to string template values.
            The keys become output fields on the interface.
            The templates should use {}-formatting syntax, where
            the names in curly braces become inputs fields on the interface.
            Format strings can also use glob wildcards to match multiple
            files. At runtime, the values of the interface inputs will be
            plugged into these templates, and the resulting strings will be
            used to select files.

        """
        super(SelectFiles, self).__init__(**kwargs)

        # Infer the infields and outfields from the template
        infields = []
        for name, template in templates.iteritems():
            for _, field_name, _, _ in string.Formatter().parse(template):
                if field_name is not None and field_name not in infields:
                    infields.append(field_name)

        self._infields = infields
        self._outfields = list(templates)
        self._templates = templates

        # Add the dynamic input fields
        undefined_traits = {}
        for field in infields:
            self.inputs.add_trait(field, traits.Any)
            undefined_traits[field] = Undefined
        self.inputs.trait_set(trait_change_notify=False, **undefined_traits)

    def _add_output_traits(self, base):
        """Add the dynamic output fields"""
        return add_traits(base, self._templates.keys())

    def _list_outputs(self):
        """Find the files and expose them as interface outputs."""
        outputs = {}
        info = dict([(k, v) for k, v in self.inputs.__dict__.items()
                     if k in self._infields])

        force_lists = self.inputs.force_lists
        if isinstance(force_lists, bool):
            force_lists = self._outfields if force_lists else []
        bad_fields = set(force_lists) - set(self._outfields)
        if bad_fields:
            bad_fields = ", ".join(list(bad_fields))
            plural = "s" if len(bad_fields) > 1 else ""
            verb = "were" if len(bad_fields) > 1 else "was"
            msg = ("The field%s '%s' %s set in 'force_lists' and not in "
                   "'templates'.") % (plural, bad_fields, verb)
            raise ValueError(msg)

        for field, template in self._templates.iteritems():

            # Build the full template path
            if isdefined(self.inputs.base_directory):
                template = op.abspath(op.join(
                    self.inputs.base_directory, template))
            else:
                template = op.abspath(template)

            # Fill in the template and glob for files
            filled_template = template.format(**info)
            filelist = glob.glob(filled_template)

            # Handle the case where nothing matched
            if not filelist:
                msg = "No files were found matching %s template: %s" % (
                    field, filled_template)
                if self.inputs.raise_on_empty:
                    raise IOError(msg)
                else:
                    warn(msg)

            # Possibly sort the list
            if self.inputs.sort_filelist:
                filelist = human_order_sorted(filelist)

            # Handle whether this must be a list or not
            if field not in force_lists:
                filelist = list_to_filename(filelist)

            outputs[field] = filelist

        return outputs


class DataFinderInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
    root_paths = traits.Either(traits.List(),
                               traits.Str(),
                               mandatory=True,)
    match_regex = traits.Str('(.+)',
                             usedefault=True,
                             desc=("Regular expression for matching "
                             "paths."))
    ignore_regexes = traits.List(desc=("List of regular expressions, "
                                 "if any match the path it will be "
                                 "ignored.")
                                 )
    max_depth = traits.Int(desc="The maximum depth to search beneath "
                           "the root_paths")
    min_depth = traits.Int(desc="The minimum depth to search beneath "
                           "the root paths")
    unpack_single = traits.Bool(False,
                                usedefault=True,
                                desc="Unpack single results from list")


class DataFinder(IOBase):
    """Search for paths that match a given regular expression. Allows a less
    proscriptive approach to gathering input files compared to DataGrabber.
    Will recursively search any subdirectories by default. This can be limited
    with the min/max depth options.
    Matched paths are available in the output 'out_paths'. Any named groups of
    captured text from the regular expression are also available as ouputs of
    the same name.

    Examples
    --------

    >>> from nipype.interfaces.io import DataFinder
    >>> df = DataFinder()
    >>> df.inputs.root_paths = '.'
    >>> df.inputs.match_regex = '.+/(?P<series_dir>.+(qT1|ep2d_fid_T1).+)/(?P<basename>.+)\.nii.gz'
    >>> result = df.run() # doctest: +SKIP
    >>> print result.outputs.out_paths # doctest: +SKIP
    ['./027-ep2d_fid_T1_Gd4/acquisition.nii.gz',
     './018-ep2d_fid_T1_Gd2/acquisition.nii.gz',
     './016-ep2d_fid_T1_Gd1/acquisition.nii.gz',
     './013-ep2d_fid_T1_pre/acquisition.nii.gz']
    >>> print result.outputs.series_dir # doctest: +SKIP
    ['027-ep2d_fid_T1_Gd4',
     '018-ep2d_fid_T1_Gd2',
     '016-ep2d_fid_T1_Gd1',
     '013-ep2d_fid_T1_pre']
    >>> print result.outputs.basename # doctest: +SKIP
    ['acquisition',
     'acquisition'
     'acquisition',
     'acquisition']

    """

    input_spec = DataFinderInputSpec
    output_spec = DynamicTraitedSpec
    _always_run = True

    def _match_path(self, target_path):
        #Check if we should ignore the path
        for ignore_re in self.ignore_regexes:
            if ignore_re.search(target_path):
                return
        #Check if we can match the path
        match = self.match_regex.search(target_path)
        if not match is None:
            match_dict = match.groupdict()
            if self.result is None:
                self.result = {'out_paths': []}
                for key in match_dict.keys():
                    self.result[key] = []
            self.result['out_paths'].append(target_path)
            for key, val in match_dict.iteritems():
                self.result[key].append(val)

    def _run_interface(self, runtime):
        #Prepare some of the inputs
        if isinstance(self.inputs.root_paths, six.string_types):
            self.inputs.root_paths = [self.inputs.root_paths]
        self.match_regex = re.compile(self.inputs.match_regex)
        if self.inputs.max_depth is Undefined:
            max_depth = None
        else:
            max_depth = self.inputs.max_depth
        if self.inputs.min_depth is Undefined:
            min_depth = 0
        else:
            min_depth = self.inputs.min_depth
        if self.inputs.ignore_regexes is Undefined:
            self.ignore_regexes = []
        else:
            self.ignore_regexes = \
                [re.compile(regex)
                 for regex in self.inputs.ignore_regexes]
        self.result = None
        for root_path in self.inputs.root_paths:
            #Handle tilda/env variables and remove extra seperators
            root_path = os.path.normpath(os.path.expandvars(os.path.expanduser(root_path)))
            #Check if the root_path is a file
            if os.path.isfile(root_path):
                if min_depth == 0:
                    self._match_path(root_path)
                continue
            #Walk through directory structure checking paths
            for curr_dir, sub_dirs, files in os.walk(root_path):
                #Determine the current depth from the root_path
                curr_depth = (curr_dir.count(os.sep) -
                              root_path.count(os.sep))
                #If the max path depth has been reached, clear sub_dirs
                #and files
                if max_depth is not None and curr_depth >= max_depth:
                    sub_dirs[:] = []
                    files = []
                #Test the path for the curr_dir and all files
                if curr_depth >= min_depth:
                    self._match_path(curr_dir)
                if curr_depth >= (min_depth - 1):
                    for infile in files:
                        full_path = os.path.join(curr_dir, infile)
                        self._match_path(full_path)
        if (self.inputs.unpack_single and
            len(self.result['out_paths']) == 1
            ):
            for key, vals in self.result.iteritems():
                self.result[key] = vals[0]
        else:
            #sort all keys acording to out_paths
            for key in self.result.keys():
                if key == "out_paths":
                    continue
                sort_tuples = human_order_sorted(zip(self.result["out_paths"],
                                                     self.result[key]))
                self.result[key] = [x for (_, x) in sort_tuples]
            self.result["out_paths"] = human_order_sorted(self.result["out_paths"])

        if not self.result:
            raise RuntimeError("Regular expression did not match any files!")

        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs.update(self.result)
        return outputs


class FSSourceInputSpec(BaseInterfaceInputSpec):
    subjects_dir = Directory(mandatory=True,
                             desc='Freesurfer subjects directory.')
    subject_id = traits.Str(mandatory=True,
                            desc='Subject name for whom to retrieve data')
    hemi = traits.Enum('both', 'lh', 'rh', usedefault=True,
                       desc='Selects hemisphere specific outputs')


class FSSourceOutputSpec(TraitedSpec):
    T1 = File(
        exists=True, desc='Intensity normalized whole-head volume', loc='mri')
    aseg = File(
        exists=True, desc='Volumetric map of regions from automatic segmentation',
        loc='mri')
    brain = File(
        exists=True, desc='Intensity normalized brain-only volume', loc='mri')
    brainmask = File(
        exists=True, desc='Skull-stripped (brain-only) volume', loc='mri')
    filled = File(exists=True, desc='Subcortical mass volume', loc='mri')
    norm = File(
        exists=True, desc='Normalized skull-stripped volume', loc='mri')
    nu = File(exists=True,
              desc='Non-uniformity corrected whole-head volume', loc='mri')
    orig = File(exists=True, desc='Base image conformed to Freesurfer space',
                loc='mri')
    rawavg = File(exists=True, desc='Volume formed by averaging input images',
                  loc='mri')
    ribbon = OutputMultiPath(
        File(exists=True), desc='Volumetric maps of cortical ribbons',
        loc='mri', altkey='*ribbon')
    wm = File(exists=True, desc='Segmented white-matter volume', loc='mri')
    wmparc = File(
        exists=True, desc='Aparc parcellation projected into subcortical white matter',
        loc='mri')
    curv = OutputMultiPath(File(exists=True), desc='Maps of surface curvature',
                           loc='surf')
    inflated = OutputMultiPath(
        File(exists=True), desc='Inflated surface meshes',
        loc='surf')
    pial = OutputMultiPath(
        File(exists=True), desc='Gray matter/pia mater surface meshes',
        loc='surf')
    smoothwm = OutputMultiPath(File(exists=True), loc='surf',
                               desc='Smoothed original surface meshes')
    sphere = OutputMultiPath(
        File(exists=True), desc='Spherical surface meshes',
        loc='surf')
    sulc = OutputMultiPath(
        File(exists=True), desc='Surface maps of sulcal depth', loc='surf')
    thickness = OutputMultiPath(File(exists=True), loc='surf',
                                desc='Surface maps of cortical thickness')
    volume = OutputMultiPath(
        File(exists=True), desc='Surface maps of cortical volume', loc='surf')
    white = OutputMultiPath(
        File(exists=True), desc='White/gray matter surface meshes',
        loc='surf')
    label = OutputMultiPath(
        File(exists=True), desc='Volume and surface label files',
        loc='label', altkey='*label')
    annot = OutputMultiPath(File(exists=True), desc='Surface annotation files',
                            loc='label', altkey='*annot')
    aparc_aseg = OutputMultiPath(
        File(exists=True), loc='mri', altkey='aparc*aseg',
        desc='Aparc parcellation projected into aseg volume')
    sphere_reg = OutputMultiPath(
        File(exists=True), loc='surf', altkey='sphere.reg',
        desc='Spherical registration file')
    aseg_stats = OutputMultiPath(File(exists=True), loc='stats', altkey='aseg',
                                 desc='Automated segmentation statistics file')
    wmparc_stats = OutputMultiPath(
        File(exists=True), loc='stats', altkey='wmparc',
        desc='White matter parcellation statistics file')
    aparc_stats = OutputMultiPath(
        File(exists=True), loc='stats', altkey='aparc',
        desc='Aparc parcellation statistics files')
    BA_stats = OutputMultiPath(File(exists=True), loc='stats', altkey='BA',
                               desc='Brodmann Area statistics files')
    aparc_a2009s_stats = OutputMultiPath(
        File(exists=True), loc='stats', altkey='aparc.a2009s',
        desc='Aparc a2009s parcellation statistics files')
    curv_stats = OutputMultiPath(File(exists=True), loc='stats', altkey='curv',
                                 desc='Curvature statistics files')
    entorhinal_exvivo_stats = OutputMultiPath(
        File(exists=True), loc='stats', altkey='entorhinal_exvivo',
        desc='Entorhinal exvivo statistics files')


class FreeSurferSource(IOBase):
    """Generates freesurfer subject info from their directories

    Examples
    --------

    >>> from nipype.interfaces.io import FreeSurferSource
    >>> fs = FreeSurferSource()
    >>> #fs.inputs.subjects_dir = '.'
    >>> fs.inputs.subject_id = 'PWS04'
    >>> res = fs.run() # doctest: +SKIP

    >>> fs.inputs.hemi = 'lh'
    >>> res = fs.run() # doctest: +SKIP

    """
    input_spec = FSSourceInputSpec
    output_spec = FSSourceOutputSpec
    _always_run = True
    _additional_metadata = ['loc', 'altkey']

    def _get_files(self, path, key, dirval, altkey=None):
        globsuffix = ''
        if dirval == 'mri':
            globsuffix = '.mgz'
        elif dirval == 'stats':
            globsuffix = '.stats'
        globprefix = ''
        if key == 'ribbon' or dirval in ['surf', 'label', 'stats']:
            if self.inputs.hemi != 'both':
                globprefix = self.inputs.hemi + '.'
            else:
                globprefix = '*'
        if key == 'aseg_stats' or key == 'wmparc_stats':
            globprefix = ''
        keydir = os.path.join(path, dirval)
        if altkey:
            key = altkey
        globpattern = os.path.join(
            keydir, ''.join((globprefix, key, globsuffix)))
        return glob.glob(globpattern)

    def _list_outputs(self):
        subjects_dir = self.inputs.subjects_dir
        subject_path = os.path.join(subjects_dir, self.inputs.subject_id)
        output_traits = self._outputs()
        outputs = output_traits.get()
        for k in outputs.keys():
            val = self._get_files(subject_path, k,
                                  output_traits.traits()[k].loc,
                                  output_traits.traits()[k].altkey)
            if val:
                outputs[k] = list_to_filename(val)
        return outputs


class XNATSourceInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):

    query_template = traits.Str(
        mandatory=True,
        desc=('Layout used to get files. Relative to base '
              'directory if defined')
    )

    query_template_args = traits.Dict(
        traits.Str,
        traits.List(traits.List),
        value=dict(outfiles=[]), usedefault=True,
        desc='Information to plug into template'
    )

    server = traits.Str(
        mandatory=True,
        requires=['user', 'pwd'],
        xor=['config']
    )

    user = traits.Str()
    pwd = traits.Password()
    config = File(mandatory=True, xor=['server'])

    cache_dir = Directory(desc='Cache directory')


class XNATSource(IOBase):
    """ Generic XNATSource module that wraps around the pyxnat module in
        an intelligent way for neuroimaging tasks to grab files and data
        from an XNAT server.

        Examples
        --------

        >>> from nipype.interfaces.io import XNATSource

        Pick all files from current directory

        >>> dg = XNATSource()
        >>> dg.inputs.template = '*'

        >>> dg = XNATSource(infields=['project','subject','experiment','assessor','inout'])
        >>> dg.inputs.query_template = '/projects/%s/subjects/%s/experiments/%s' \
                   '/assessors/%s/%s_resources/files'
        >>> dg.inputs.project = 'IMAGEN'
        >>> dg.inputs.subject = 'IMAGEN_000000001274'
        >>> dg.inputs.experiment = '*SessionA*'
        >>> dg.inputs.assessor = '*ADNI_MPRAGE_nii'
        >>> dg.inputs.inout = 'out'

        >>> dg = XNATSource(infields=['sid'],outfields=['struct','func'])
        >>> dg.inputs.query_template = '/projects/IMAGEN/subjects/%s/experiments/*SessionA*' \
                   '/assessors/*%s_nii/out_resources/files'
        >>> dg.inputs.query_template_args['struct'] = [['sid','ADNI_MPRAGE']]
        >>> dg.inputs.query_template_args['func'] = [['sid','EPI_faces']]
        >>> dg.inputs.sid = 'IMAGEN_000000001274'


    """
    input_spec = XNATSourceInputSpec
    output_spec = DynamicTraitedSpec

    def __init__(self, infields=None, outfields=None, **kwargs):
        """
        Parameters
        ----------
        infields : list of str
            Indicates the input fields to be dynamically created

        outfields: list of str
            Indicates output fields to be dynamically created

        See class examples for usage

        """
        super(XNATSource, self).__init__(**kwargs)
        undefined_traits = {}
        # used for mandatory inputs check
        self._infields = infields
        if infields:
            for key in infields:
                self.inputs.add_trait(key, traits.Any)
                undefined_traits[key] = Undefined
            self.inputs.query_template_args['outfiles'] = [infields]
        if outfields:
            # add ability to insert field specific templates
            self.inputs.add_trait(
                'field_template',
                traits.Dict(traits.Enum(outfields),
                            desc="arguments that fit into query_template")
            )
            undefined_traits['field_template'] = Undefined
            #self.inputs.remove_trait('query_template_args')
            outdict = {}
            for key in outfields:
                outdict[key] = []
            self.inputs.query_template_args = outdict
        self.inputs.trait_set(trait_change_notify=False, **undefined_traits)

    def _add_output_traits(self, base):
        """

        Using traits.Any instead out OutputMultiPath till add_trait bug
        is fixed.
        """
        return add_traits(base, self.inputs.query_template_args.keys())

    def _list_outputs(self):
        # infields are mandatory, however I could not figure out
        # how to set 'mandatory' flag dynamically, hence manual check

        cache_dir = self.inputs.cache_dir or tempfile.gettempdir()

        if self.inputs.config:
            xnat = pyxnat.Interface(config=self.inputs.config)
        else:
            xnat = pyxnat.Interface(self.inputs.server,
                                    self.inputs.user,
                                    self.inputs.pwd,
                                    cache_dir
                                    )

        if self._infields:
            for key in self._infields:
                value = getattr(self.inputs, key)
                if not isdefined(value):
                    msg = ("%s requires a value for input '%s' "
                           "because it was listed in 'infields'" %
                          (self.__class__.__name__, key)
                           )
                    raise ValueError(msg)

        outputs = {}
        for key, args in self.inputs.query_template_args.items():
            outputs[key] = []
            template = self.inputs.query_template
            if hasattr(self.inputs, 'field_template') and \
                    isdefined(self.inputs.field_template) and \
                    key in self.inputs.field_template:
                template = self.inputs.field_template[key]
            if not args:
                file_objects = xnat.select(template).get('obj')
                if file_objects == []:
                    raise IOError('Template %s returned no files'
                                  % template
                                  )
                outputs[key] = list_to_filename(
                    [str(file_object.get())
                     for file_object in file_objects
                     if file_object.exists()
                     ])
            for argnum, arglist in enumerate(args):
                maxlen = 1
                for arg in arglist:
                    if isinstance(arg, six.string_types) and hasattr(self.inputs, arg):
                        arg = getattr(self.inputs, arg)
                    if isinstance(arg, list):
                        if (maxlen > 1) and (len(arg) != maxlen):
                            raise ValueError('incompatible number '
                                             'of arguments for %s' % key
                                             )
                        if len(arg) > maxlen:
                            maxlen = len(arg)
                outfiles = []
                for i in range(maxlen):
                    argtuple = []
                    for arg in arglist:
                        if isinstance(arg, six.string_types) and \
                                hasattr(self.inputs, arg):
                            arg = getattr(self.inputs, arg)
                        if isinstance(arg, list):
                            argtuple.append(arg[i])
                        else:
                            argtuple.append(arg)
                    if argtuple:
                        target = template % tuple(argtuple)
                        file_objects = xnat.select(target).get('obj')

                        if file_objects == []:
                            raise IOError('Template %s '
                                          'returned no files' % target
                                          )

                        outfiles = list_to_filename(
                            [str(file_object.get())
                             for file_object in file_objects
                             if file_object.exists()
                             ]
                        )
                    else:
                        file_objects = xnat.select(template).get('obj')

                        if file_objects == []:
                            raise IOError('Template %s '
                                          'returned no files' % template
                                          )

                        outfiles = list_to_filename(
                            [str(file_object.get())
                             for file_object in file_objects
                             if file_object.exists()
                             ]
                        )

                    outputs[key].insert(i, outfiles)
            if len(outputs[key]) == 0:
                outputs[key] = None
            elif len(outputs[key]) == 1:
                outputs[key] = outputs[key][0]
        return outputs


class XNATSinkInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):

    _outputs = traits.Dict(traits.Str, value={}, usedefault=True)

    server = traits.Str(mandatory=True,
                        requires=['user', 'pwd'],
                        xor=['config']
                        )

    user = traits.Str()
    pwd = traits.Password()
    config = File(mandatory=True, xor=['server'])
    cache_dir = Directory(desc='')

    project_id = traits.Str(
        desc='Project in which to store the outputs', mandatory=True)

    subject_id = traits.Str(
        desc='Set to subject id', mandatory=True)

    experiment_id = traits.Str(
        desc='Set to workflow name', mandatory=True)

    assessor_id = traits.Str(
        desc=('Option to customize ouputs representation in XNAT - '
              'assessor level will be used with specified id'),
        xor=['reconstruction_id']
    )

    reconstruction_id = traits.Str(
        desc=('Option to customize ouputs representation in XNAT - '
              'reconstruction level will be used with specified id'),
        xor=['assessor_id']
    )

    share = traits.Bool(False,
        desc=('Option to share the subjects from the original project'
              'instead of creating new ones when possible - the created '
              'experiments are then shared back to the original project'
              ),
        usedefault=True)

    def __setattr__(self, key, value):
        if key not in self.copyable_trait_names():
            self._outputs[key] = value
        else:
            super(XNATSinkInputSpec, self).__setattr__(key, value)


class XNATSink(IOBase):
    """ Generic datasink module that takes a directory containing a
        list of nifti files and provides a set of structured output
        fields.
    """
    input_spec = XNATSinkInputSpec

    def _list_outputs(self):
        """Execute this module.
        """

        # setup XNAT connection
        cache_dir = self.inputs.cache_dir or tempfile.gettempdir()

        if self.inputs.config:
            xnat = pyxnat.Interface(config=self.inputs.config)
        else:
            xnat = pyxnat.Interface(self.inputs.server,
                                    self.inputs.user,
                                    self.inputs.pwd,
                                    cache_dir
                                    )

        # if possible share the subject from the original project
        if self.inputs.share:
            subject_id = self.inputs.subject_id
            result = xnat.select(
                'xnat:subjectData',
                ['xnat:subjectData/PROJECT',
                 'xnat:subjectData/SUBJECT_ID']
            ).where('xnat:subjectData/SUBJECT_ID = %s AND' % subject_id)

            # subject containing raw data exists on the server
            if (result.data and isinstance(result.data[0], dict)):
                result = result.data[0]
                shared = xnat.select('/project/%s/subject/%s' %
                                     (self.inputs.project_id,
                                      self.inputs.subject_id
                                      )
                                     )

                if not shared.exists():  # subject not in share project

                    share_project = xnat.select(
                        '/project/%s' % self.inputs.project_id)

                    if not share_project.exists():  # check project exists
                        share_project.insert()

                    subject = xnat.select('/project/%(project)s'
                                          '/subject/%(subject_id)s' % result
                                          )

                    subject.share(str(self.inputs.project_id))

        # setup XNAT resource
        uri_template_args = dict(
            project_id=quote_id(self.inputs.project_id),
            subject_id=self.inputs.subject_id,
            experiment_id=quote_id(self.inputs.experiment_id))

        if self.inputs.share:
            uri_template_args['original_project'] = result['project']

        if self.inputs.assessor_id:
            uri_template_args['assessor_id'] = quote_id(self.inputs.assessor_id)
        elif self.inputs.reconstruction_id:
            uri_template_args['reconstruction_id'] = quote_id(self.inputs.reconstruction_id)

        # gather outputs and upload them
        for key, files in self.inputs._outputs.items():

            for name in filename_to_list(files):

                if isinstance(name, list):
                    for i, file_name in enumerate(name):
                        push_file(self, xnat, file_name,
                                  '%s_' % i + key,
                                  uri_template_args
                                  )
                else:
                    push_file(self, xnat, name, key, uri_template_args)


def quote_id(string):
    return str(string).replace('_', '---')


def unquote_id(string):
    return str(string).replace('---', '_')


def push_file(self, xnat, file_name, out_key, uri_template_args):

    # grab info from output file names
    val_list = [unquote_id(val)
                for part in os.path.split(file_name)[0].split(os.sep)
                for val in part.split('_')[1:]
                if part.startswith('_') and len(part.split('_')) % 2
                ]

    keymap = dict(zip(val_list[1::2], val_list[2::2]))

    _label = []
    for key, val in sorted(keymap.items()):
        if str(self.inputs.subject_id) not in val:
            _label.extend([key, val])

    # select and define container level
    uri_template_args['container_type'] = None

    for container in ['assessor_id', 'reconstruction_id']:
        if getattr(self.inputs, container):
            uri_template_args['container_type'] = container.split('_id')[0]
            uri_template_args['container_id'] = uri_template_args[container]

    if uri_template_args['container_type'] is None:
        uri_template_args['container_type'] = 'reconstruction'

        uri_template_args['container_id'] = unquote_id(
            uri_template_args['experiment_id']
        )

        if _label:
            uri_template_args['container_id'] += (
                '_results_%s' % '_'.join(_label)
            )
        else:
            uri_template_args['container_id'] += '_results'

    # define resource level
    uri_template_args['resource_label'] = (
        '%s_%s' % (uri_template_args['container_id'],
                   out_key.split('.')[0]
                   )
    )

    # define file level
    uri_template_args['file_name'] = os.path.split(
        os.path.abspath(unquote_id(file_name)))[1]

    uri_template = (
        '/project/%(project_id)s/subject/%(subject_id)s'
        '/experiment/%(experiment_id)s/%(container_type)s/%(container_id)s'
        '/out/resource/%(resource_label)s/file/%(file_name)s'
    )

    # unquote values before uploading
    for key in uri_template_args.keys():
        uri_template_args[key] = unquote_id(uri_template_args[key])

    # upload file
    remote_file = xnat.select(uri_template % uri_template_args)
    remote_file.insert(file_name,
                       experiments='xnat:imageSessionData',
                       use_label=True
                       )

    # shares the experiment back to the original project if relevant
    if 'original_project' in uri_template_args:

        experiment_template = (
            '/project/%(original_project)s'
            '/subject/%(subject_id)s/experiment/%(experiment_id)s'
        )

        xnat.select(experiment_template % uri_template_args
                    ).share(uri_template_args['original_project'])


def capture_provenance():
    pass


def push_provenance():
    pass


class SQLiteSinkInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
    database_file = File(exists=True, mandatory=True)
    table_name = traits.Str(mandatory=True)


class SQLiteSink(IOBase):
    """ Very simple frontend for storing values into SQLite database.

        .. warning::

            This is not a thread-safe node because it can write to a common
            shared location. It will not complain when it overwrites a file.

        Examples
        --------

        >>> sql = SQLiteSink(input_names=['subject_id', 'some_measurement'])
        >>> sql.inputs.database_file = 'my_database.db'
        >>> sql.inputs.table_name = 'experiment_results'
        >>> sql.inputs.subject_id = 's1'
        >>> sql.inputs.some_measurement = 11.4
        >>> sql.run() # doctest: +SKIP

    """
    input_spec = SQLiteSinkInputSpec

    def __init__(self, input_names, **inputs):

        super(SQLiteSink, self).__init__(**inputs)

        self._input_names = filename_to_list(input_names)
        add_traits(self.inputs, [name for name in self._input_names])

    def _list_outputs(self):
        """Execute this module.
        """
        conn = sqlite3.connect(self.inputs.database_file,
                               check_same_thread=False)
        c = conn.cursor()
        c.execute("INSERT OR REPLACE INTO %s (" % self.inputs.table_name +
                  ",".join(self._input_names) + ") VALUES (" +
                  ",".join(["?"] * len(self._input_names)) + ")",
                  [getattr(self.inputs, name) for name in self._input_names])
        conn.commit()
        c.close()
        return None


class MySQLSinkInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
    host = traits.Str('localhost', mandatory=True,
                      requires=['username', 'password'],
                      xor=['config'], usedefault=True)
    config = File(mandatory=True, xor=['host'],
                  desc="MySQL Options File (same format as my.cnf)")
    database_name = traits.Str(
        mandatory=True, desc='Otherwise known as the schema name')
    table_name = traits.Str(mandatory=True)
    username = traits.Str()
    password = traits.Str()


class MySQLSink(IOBase):
    """ Very simple frontend for storing values into MySQL database.

        Examples
        --------

        >>> sql = MySQLSink(input_names=['subject_id', 'some_measurement'])
        >>> sql.inputs.database_name = 'my_database'
        >>> sql.inputs.table_name = 'experiment_results'
        >>> sql.inputs.username = 'root'
        >>> sql.inputs.password = 'secret'
        >>> sql.inputs.subject_id = 's1'
        >>> sql.inputs.some_measurement = 11.4
        >>> sql.run() # doctest: +SKIP

    """
    input_spec = MySQLSinkInputSpec

    def __init__(self, input_names, **inputs):

        super(MySQLSink, self).__init__(**inputs)

        self._input_names = filename_to_list(input_names)
        add_traits(self.inputs, [name for name in self._input_names])

    def _list_outputs(self):
        """Execute this module.
        """
        import MySQLdb
        if isdefined(self.inputs.config):
            conn = MySQLdb.connect(db=self.inputs.database_name,
                                   read_default_file=self.inputs.config)
        else:
            conn = MySQLdb.connect(host=self.inputs.host,
                                   user=self.inputs.username,
                                   passwd=self.inputs.password,
                                   db=self.inputs.database_name)
        c = conn.cursor()
        c.execute("REPLACE INTO %s (" % self.inputs.table_name +
                  ",".join(self._input_names) + ") VALUES (" +
                  ",".join(["%s"] * len(self._input_names)) + ")",
                  [getattr(self.inputs, name) for name in self._input_names])
        conn.commit()
        c.close()
        return None

class SSHDataGrabberInputSpec(DataGrabberInputSpec):
    hostname = traits.Str(mandatory=True, desc='Server hostname.')
    username = traits.Str(desc='Server username.')
    password = traits.Password(desc='Server password.')
    download_files = traits.Bool(True, usedefault=True,
                                    desc='If false it will return the file names without downloading them')
    base_directory = traits.Str(mandatory=True,
                               desc='Path to the base directory consisting of subject data.')
    template_expression = traits.Enum(['fnmatch', 'regexp'], usedefault=True,
                            desc='Use either fnmatch or regexp to express templates')
    ssh_log_to_file = traits.Str('', usedefault=True,
                            desc='If set SSH commands will be logged to the given file')


class SSHDataGrabber(DataGrabber):
    """ Extension of DataGrabber module that downloads the file list and
        optionally the files from a SSH server. The SSH operation must
        not need user and password so an SSH agent must be active in
        where this module is being run.


        .. attention::

           Doesn't support directories currently

        Examples
        --------

        >>> from nipype.interfaces.io import SSHDataGrabber
        >>> dg = SSHDataGrabber()
        >>> dg.inputs.hostname = 'test.rebex.net'
        >>> dg.inputs.user = 'demo'
        >>> dg.inputs.password = 'password'
        >>> dg.inputs.base_directory = 'pub/example'

        Pick all files from the base directory

        >>> dg.inputs.template = '*'

        Pick all files starting with "s" and a number from current directory

        >>> dg.inputs.template_expression = 'regexp'
        >>> dg.inputs.template = 'pop[0-9].*'

        Same thing but with dynamically created fields

        >>> dg = SSHDataGrabber(infields=['arg1','arg2'])
        >>> dg.inputs.hostname = 'test.rebex.net'
        >>> dg.inputs.user = 'demo'
        >>> dg.inputs.password = 'password'
        >>> dg.inputs.base_directory = 'pub'
        >>> dg.inputs.template = '%s/%s.txt'
        >>> dg.inputs.arg1 = 'example'
        >>> dg.inputs.arg2 = 'foo'

        however this latter form can be used with iterables and iterfield in a
        pipeline.

        Dynamically created, user-defined input and output fields

        >>> dg = SSHDataGrabber(infields=['sid'], outfields=['func','struct','ref'])
        >>> dg.inputs.hostname = 'myhost.com'
        >>> dg.inputs.base_directory = '/main_folder/my_remote_dir'
        >>> dg.inputs.template_args['func'] = [['sid',['f3','f5']]]
        >>> dg.inputs.template_args['struct'] = [['sid',['struct']]]
        >>> dg.inputs.template_args['ref'] = [['sid','ref']]
        >>> dg.inputs.sid = 's1'

        Change the template only for output field struct. The rest use the
        general template

        >>> dg.inputs.field_template = dict(struct='%s/struct.nii')
        >>> dg.inputs.template_args['struct'] = [['sid']]

    """
    input_spec = SSHDataGrabberInputSpec
    output_spec = DynamicTraitedSpec
    _always_run = False

    def __init__(self, infields=None, outfields=None, **kwargs):
        """
        Parameters
        ----------
        infields : list of str
            Indicates the input fields to be dynamically created

        outfields: list of str
            Indicates output fields to be dynamically created

        See class examples for usage

        """
        try:
            paramiko
        except NameError:
            warn(
                "The library parmiko needs to be installed"
                " for this module to run."
            )
        if not outfields:
            outfields = ['outfiles']
        kwargs = kwargs.copy()
        kwargs['infields'] = infields
        kwargs['outfields'] = outfields
        super(SSHDataGrabber, self).__init__(**kwargs)
        if (
            None in (self.inputs.username, self.inputs.password)
        ):
            raise ValueError(
                "either both username and password "
                "are provided or none of them"
            )

        if (
            self.inputs.template_expression == 'regexp' and
            self.inputs.template[-1] != '$'
        ):
            self.inputs.template += '$'


    def _list_outputs(self):
        try:
            paramiko
        except NameError:
            raise ImportError(
                "The library parmiko needs to be installed"
                " for this module to run."
            )

        if len(self.inputs.ssh_log_to_file) > 0:
            paramiko.util.log_to_file(self.inputs.ssh_log_to_file)
        # infields are mandatory, however I could not figure out how to set 'mandatory' flag dynamically
        # hence manual check
        if self._infields:
            for key in self._infields:
                value = getattr(self.inputs, key)
                if not isdefined(value):
                    msg = "%s requires a value for input '%s' because it was listed in 'infields'" % \
                        (self.__class__.__name__, key)
                    raise ValueError(msg)

        outputs = {}
        for key, args in self.inputs.template_args.items():
            outputs[key] = []
            template = self.inputs.template
            if hasattr(self.inputs, 'field_template') and \
                    isdefined(self.inputs.field_template) and \
                    key in self.inputs.field_template:
                template = self.inputs.field_template[key]
            if not args:
                client = self._get_ssh_client()
                sftp = client.open_sftp()
                sftp.chdir(self.inputs.base_directory)
                filelist = sftp.listdir()
                if self.inputs.template_expression == 'fnmatch':
                    filelist = fnmatch.filter(filelist, template)
                elif self.inputs.template_expression == 'regexp':
                    regexp = re.compile(template)
                    filelist = filter(regexp.match, filelist)
                else:
                    raise ValueError('template_expression value invalid')
                if len(filelist) == 0:
                    msg = 'Output key: %s Template: %s returned no files' % (
                        key, template)
                    if self.inputs.raise_on_empty:
                        raise IOError(msg)
                    else:
                        warn(msg)
                else:
                    if self.inputs.sort_filelist:
                        filelist = human_order_sorted(filelist)
                    outputs[key] = list_to_filename(filelist)
                if self.inputs.download_files:
                    for f in filelist:
                        sftp.get(f, f)
            for argnum, arglist in enumerate(args):
                maxlen = 1
                for arg in arglist:
                    if isinstance(arg, six.string_types) and hasattr(self.inputs, arg):
                        arg = getattr(self.inputs, arg)
                    if isinstance(arg, list):
                        if (maxlen > 1) and (len(arg) != maxlen):
                            raise ValueError('incompatible number of arguments for %s' % key)
                        if len(arg) > maxlen:
                            maxlen = len(arg)
                outfiles = []
                for i in range(maxlen):
                    argtuple = []
                    for arg in arglist:
                        if isinstance(arg, six.string_types) and hasattr(self.inputs, arg):
                            arg = getattr(self.inputs, arg)
                        if isinstance(arg, list):
                            argtuple.append(arg[i])
                        else:
                            argtuple.append(arg)
                    filledtemplate = template
                    if argtuple:
                        try:
                            filledtemplate = template % tuple(argtuple)
                        except TypeError as e:
                            raise TypeError(e.message + ": Template %s failed to convert with args %s" % (template, str(tuple(argtuple))))
                    client = self._get_ssh_client()
                    sftp = client.open_sftp()
                    sftp.chdir(self.inputs.base_directory)
                    filledtemplate_dir = os.path.dirname(filledtemplate)
                    filledtemplate_base = os.path.basename(filledtemplate)
                    filelist = sftp.listdir(filledtemplate_dir)
                    if self.inputs.template_expression == 'fnmatch':
                        outfiles = fnmatch.filter(filelist, filledtemplate_base)
                    elif self.inputs.template_expression == 'regexp':
                        regexp = re.compile(filledtemplate_base)
                        outfiles = filter(regexp.match, filelist)
                    else:
                        raise ValueError('template_expression value invalid')
                    if len(outfiles) == 0:
                        msg = 'Output key: %s Template: %s returned no files' % (key, filledtemplate)
                        if self.inputs.raise_on_empty:
                            raise IOError(msg)
                        else:
                            warn(msg)
                        outputs[key].append(None)
                    else:
                        if self.inputs.sort_filelist:
                            outfiles = human_order_sorted(outfiles)
                        outputs[key].append(list_to_filename(outfiles))
                        if self.inputs.download_files:
                            for f in outfiles:
                                sftp.get(os.path.join(filledtemplate_dir, f), f)
            if any([val is None for val in outputs[key]]):
                outputs[key] = []
            if len(outputs[key]) == 0:
                outputs[key] = None
            elif len(outputs[key]) == 1:
                outputs[key] = outputs[key][0]
        return outputs

    def _get_ssh_client(self):
        config = paramiko.SSHConfig()
        config.parse(open(os.path.expanduser('~/.ssh/config')))
        host = config.lookup(self.inputs.hostname)
        if 'proxycommand' in host:
            proxy = paramiko.ProxyCommand(
                subprocess.check_output(
                    [os.environ['SHELL'], '-c', 'echo %s' % host['proxycommand']]
                ).strip()
            )
        else:
            proxy = None
        client = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        client.connect(host['hostname'], username=host['user'], sock=proxy)
        return client
