getfiles

GETFILES  Fetch new and updated files from a remote directory.

  Syntax:
    FILES = GETFILES(CONNECTION, OPTIONS)
    FILES = GETFILES(CONNECTION, OPT1, VAL1, ...)

  Description:
    FILES = GETFILES(CONNECTION, OPTIONS) and
    FILES = GETFILES(CONNECTION, OPT1, VAL1, ...) retrieve files from a 
    directory on a remote server through the FTP-like object CONNECTION 
    according to the options given in key-value pairs OPT1, VAL1... or in 
    scalar struct OPTIONS with field names as option keys and field values as 
    option values. Recognized options are:
      SOURCE: remote source directory.
       String with the name of the remote directory to download the files from.
       If not given, the current working directory on the server is used.
       Default value: [] (use remote current working directory)
     TARGET: local target directory.
       String with the name of the local directory to download the files to.
       If empty, the current working directory on the local client is used.
       Default value: [] (use local current remote directory)
     INCLUDE: name pattern of files to include in the download.
       String with the pattern (regular expression) of the files to download.
       Only files whose name match this pattern are downloaded.
       If not given, all files in the source directory are downloaded.
       Default value: [] (download all files in source directory)
     EXCLUDE: name pattern of files to exclude from the download.
       String with the pattern (regular expression) of the files to exclude.
       All files whose name match this pattern are not downloaded.
       If not given, all files are downloaded.
       Default value: [] (do not exclude any file)
     NEW: filter new files on the remote server.
       Name or handle of the predicate function new files on the server must
       satisfy to be included in the download. The function receives a single
       input with the attributes of the new files as returned by DIR operation,
       and returns one logical output whether to download respective file.
       If not given, all new files are downloaded.
       Default value: [] (download all new files)
     UPDATE: filter files on the server already existing at the local side.
       Name or handle of the predicate function files on the server must
       satisfy to consider them as updated and include them in the download.
       The function receives two inputs: the attributes of corresponding local
       and remote files as returned by DIR operation, and returns one output 
       whether to update the local file with the remote one.
       If not given, the test is based on the modification time, and only
       files on the server newer than respective local files are downloaded.
      Default value: [] (overwrite all exisiting files)

  Examples:
    connection = ftp('ftp://myserver.org')
    % Download all files in remote directory to local directory.
    files = getfiles(connection)
    % Download files in remote directory matching given name,
    % and update existing files only when the remote ones are newer.
    files = getfiles( ...
      connection, ...
      'source', '/var/opt/gmc/gliders/happyglider/from-glider', ...
      'target', 'funnymission/binary', ...
      'include', '^.*\.[smdtne]bd$', ...
      'update', @(l,r)(l.datenum < r.datenum) );

  See also:
    FTP
    SFTP

  Authors:
    Joan Pau Beltran  <joanpau.beltran@socib.cat>

0001 function files = getfiles(connection, varargin)
0002 %GETFILES  Fetch new and updated files from a remote directory.
0003 %
0004 %  Syntax:
0005 %    FILES = GETFILES(CONNECTION, OPTIONS)
0006 %    FILES = GETFILES(CONNECTION, OPT1, VAL1, ...)
0007 %
0008 %  Description:
0009 %    FILES = GETFILES(CONNECTION, OPTIONS) and
0010 %    FILES = GETFILES(CONNECTION, OPT1, VAL1, ...) retrieve files from a
0011 %    directory on a remote server through the FTP-like object CONNECTION
0012 %    according to the options given in key-value pairs OPT1, VAL1... or in
0013 %    scalar struct OPTIONS with field names as option keys and field values as
0014 %    option values. Recognized options are:
0015 %      SOURCE: remote source directory.
0016 %       String with the name of the remote directory to download the files from.
0017 %       If not given, the current working directory on the server is used.
0018 %       Default value: [] (use remote current working directory)
0019 %     TARGET: local target directory.
0020 %       String with the name of the local directory to download the files to.
0021 %       If empty, the current working directory on the local client is used.
0022 %       Default value: [] (use local current remote directory)
0023 %     INCLUDE: name pattern of files to include in the download.
0024 %       String with the pattern (regular expression) of the files to download.
0025 %       Only files whose name match this pattern are downloaded.
0026 %       If not given, all files in the source directory are downloaded.
0027 %       Default value: [] (download all files in source directory)
0028 %     EXCLUDE: name pattern of files to exclude from the download.
0029 %       String with the pattern (regular expression) of the files to exclude.
0030 %       All files whose name match this pattern are not downloaded.
0031 %       If not given, all files are downloaded.
0032 %       Default value: [] (do not exclude any file)
0033 %     NEW: filter new files on the remote server.
0034 %       Name or handle of the predicate function new files on the server must
0035 %       satisfy to be included in the download. The function receives a single
0036 %       input with the attributes of the new files as returned by DIR operation,
0037 %       and returns one logical output whether to download respective file.
0038 %       If not given, all new files are downloaded.
0039 %       Default value: [] (download all new files)
0040 %     UPDATE: filter files on the server already existing at the local side.
0041 %       Name or handle of the predicate function files on the server must
0042 %       satisfy to consider them as updated and include them in the download.
0043 %       The function receives two inputs: the attributes of corresponding local
0044 %       and remote files as returned by DIR operation, and returns one output
0045 %       whether to update the local file with the remote one.
0046 %       If not given, the test is based on the modification time, and only
0047 %       files on the server newer than respective local files are downloaded.
0048 %      Default value: [] (overwrite all exisiting files)
0049 %
0050 %  Examples:
0051 %    connection = ftp('ftp://myserver.org')
0052 %    % Download all files in remote directory to local directory.
0053 %    files = getfiles(connection)
0054 %    % Download files in remote directory matching given name,
0055 %    % and update existing files only when the remote ones are newer.
0056 %    files = getfiles( ...
0057 %      connection, ...
0058 %      'source', '/var/opt/gmc/gliders/happyglider/from-glider', ...
0059 %      'target', 'funnymission/binary', ...
0060 %      'include', '^.*\.[smdtne]bd$', ...
0061 %      'update', @(l,r)(l.datenum < r.datenum) );
0062 %
0063 %  See also:
0064 %    FTP
0065 %    SFTP
0066 %
0067 %  Authors:
0068 %    Joan Pau Beltran  <joanpau.beltran@socib.cat>
0069 
0070 %  Copyright (C) 2014-2016
0071 %  ICTS SOCIB - Servei d'observacio i prediccio costaner de les Illes Balears
0072 %  <http://www.socib.es>
0073 %
0074 %  This program is free software: you can redistribute it and/or modify
0075 %  it under the terms of the GNU General Public License as published by
0076 %  the Free Software Foundation, either version 3 of the License, or
0077 %  (at your option) any later version.
0078 %
0079 %  This program is distributed in the hope that it will be useful,
0080 %  but WITHOUT ANY WARRANTY; without even the implied warranty of
0081 %  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0082 %  GNU General Public License for more details.
0083 %
0084 %  You should have received a copy of the GNU General Public License
0085 %  along with this program.  If not, see <http://www.gnu.org/licenses/>.
0086 
0087   error(nargchk(1, 13, nargin, 'struct'));
0088   
0089   
0090   %% Set options and default values.
0091   options.source = [];
0092   options.target = [];
0093   options.include = [];
0094   options.exclude = [];
0095   options.new = [];
0096   options.update = [];
0097 
0098 
0099   %% Parse optional arguments.
0100   % Get option key-value pairs in any accepted call signature.
0101   argopts = varargin;
0102   if isscalar(argopts) && isstruct(argopts{1})
0103     % Options passed as a single option struct argument:
0104     % field names are option keys and field values are option values.
0105     opt_key_list = fieldnames(argopts{1});
0106     opt_val_list = struct2cell(argopts{1});
0107   elseif mod(numel(argopts), 2) == 0
0108     % Options passed as key-value argument pairs.
0109     opt_key_list = argopts(1:2:end);
0110     opt_val_list = argopts(2:2:end);
0111   else
0112     error('glider_toolbox:getfiles:InvalidOptions', ...
0113           'Invalid optional arguments (neither key-value pairs nor struct).');
0114   end
0115   % Overwrite default options with values given in extra arguments.
0116   for opt_idx = 1:numel(opt_key_list)
0117     opt = lower(opt_key_list{opt_idx});
0118     val = opt_val_list{opt_idx};
0119     if isfield(options, opt)
0120       options.(opt) = val;
0121     else
0122       error('glider_toolbox:getfiles:InvalidOption', ...
0123             'Invalid option: %s.', opt);
0124     end
0125   end
0126   
0127   
0128   %% Parse options to more practical values.
0129   chdir = false;
0130   totarget = false;
0131   include_all = true;
0132   exclude_none = true;
0133   new_all = true;
0134   update_all = true;
0135   if ~isequal([], options.source)
0136     chdir = true;
0137     source = options.source;
0138   end
0139   if ~isequal([], options.target)
0140     totarget = true;
0141     target = options.target;
0142   end
0143   if ~isequal([], options.include)
0144     include_all = false;
0145     include = options.include;
0146   end 
0147   if ~isequal([], options.exclude)
0148     exclude_none = false;
0149     exclude = options.exclude;
0150   end
0151   if ~isequal([], options.new)
0152     new_all = false;
0153     newfunc = options.new;
0154     if ischar(newfunc)
0155       newfunc = str2func(newfunc);
0156     end
0157   end
0158   if ~isequal([], options.update)
0159     update_all = false;
0160     updatefunc = options.update;
0161     if ischar(updatefunc)
0162       updatefunc = str2func(udpatefunc);
0163     end
0164   end
0165   
0166   
0167   %% List remote files and fetch the required ones.
0168   if chdir
0169     old_pwd = cd(connection);
0170     cd(connection, source);
0171   end
0172   ratts = dir(connection, source);
0173   if totarget
0174     latts = dir(target);
0175   else
0176     latts = dir();
0177   end
0178   select = true(size(ratts));
0179   lexist = false(size(ratts));
0180   lindex = zeros(size(ratts));
0181   [lexist(:), lindex(:)] = ismember({ratts.name}', {latts.name}');
0182   if ~include_all
0183     select(select) = ...
0184       ~cellfun(@isempty, regexp({ratts(select).name}, include, 'match'));
0185   end
0186   if ~exclude_none
0187     select(select) = ...
0188       cellfun(@isempty, regexp({ratts(select).name}, exclude, 'match'));
0189   end
0190   if ~new_all
0191     select(select & ~lexist) = arrayfun(newfunc, ratts(select & ~lexist));
0192   end
0193   if ~update_all
0194     select(select & lexist) = arrayfun(updatefunc, ...
0195                                        latts(lindex(select & lexist)), ...
0196                                        ratts(select & lexist));
0197   end
0198   if (totarget)
0199     getfunc = @(name)(mget(connection, name, target));
0200   else
0201     getfunc = @(name)(mget(connection, name));
0202   end
0203   files = cellfun(getfunc, {ratts(select).name});
0204   if chdir
0205     cd(connection, old_pwd);
0206   end
0207   
0208 end