rbasis/basisgen/+Greedy/Checkpoint.m

classdef Checkpoint
  % Helper class used to store and restore @ref DataTree "data tree" objects at
  % specified checkpoints.
  %
  % This class can be used for checkpointing. The most prominent use case is
  % the ReducedModel.gen_detailed_data() method. When this method is called,
  % checkpoints are created after each bigger basis extension step at which the
  % generated detailed_data is stored. Each of these Afterwards the checkpoint can be re-read
  % from the harddrive like this: 
  % @code
  %   ids = Checkpoint.get_ids(rmodel)
  %   [detailed_data, cp] = Checkpoint.restore(rmodel, ids(1), 'latest');
  % @endcode
  %
  % Instead of 'latest' one could also have used a file index. See
  % #file_rotation_size and store() for more details on this.


  properties (Hidden=true, Constant)
    % list of values ignored for hash computation by compute_hash()
    ignorelist = {'Mmax','MM','Mstrich','ei_Mmax','Mmax','M','N','Nmax',...
                  'verbose','debug', ...
                  'enable_error_estimator'...
                 };
  end

  properties(Dependent)
    % persistent value for all Checkpoint instances controlling whether
    % checkpoint is enabled or not.
    enable_auto_restore = 1;

    enable_storing = true;
  end

  properties (Access = public)

    % maximum number of files stored in one directory in parallel.
    %
    % If this limit is reached, the oldest file is deleted before a new
    % Checkpoint.store() is executed.
    file_rotation_size = 5;

    % The level indicates which level in a data tree of DataTree.INode instances
    % can be (re)stored by this Checkpoint instance.
    %
    % Level '1' corresponds to the root element of the data tree, level '2' to
    % a child of the root, level '3' to a child of a child of the root element,
    % ...
    level      = 1;

    % A cell array of indices as returned by DataTree.INode.get_index() defining
    % the index of the stored DataTree.INode instance with respect to the root
    % element of the tree.
    %
    % @todo: Why do I not store a single vector
    data_index = {[]};

    % a cell array of arbitrary userdata stored with each DataTree.INode node.
    userdata   = {[]};

    % a cell array of id strings which describes the DataTree.INode instance or the
    % algorithm which generates it.
    ids        = {''};

    % a cell array of filenames where the DataTree.INode children are stored.
    filenames  = {''};
  end

  methods

    function cp = Checkpoint(old, index)
      % function cp = Checkpoint(old, index)
      % creates a new checkpoint
      %
      % Parameters:
      %   old: an object of type .Greedy.Checkpoint whose checkpoint data is copied or
      %        included in the newly created checkpoint.
      %   index: if this is empty, argument old is copied, otherwise a new
      %          Checkpoint of level old.level+1 is created with data fields.

      if nargin == 2
        if isempty(index)
          cp.level = old.level;
        else
          cp.level = old.level + 1;
        end
        if cp.level <= length(old.userdata)
          cp.userdata   = old.userdata;
          cp.ids        = old.ids;
          cp.data_index = old.data_index;
          cp.filenames  = old.filenames;
        else
          cp.userdata   = [ old.userdata, {[]} ];
          cp.ids        = [ old.ids, {''} ];
          cp.filenames  = [ old.filenames, {''} ];
          cp.data_index = [ old.data_index, {[old.data_index{end}, index]} ];
        end
      end
    end

    function ec = get.enable_auto_restore(this)
      ec = this.static_flags('auto_restore');
    end

    function this = set.enable_auto_restore(this, nec)
      ec = this.static_flags('auto_restore', nec);
    end

    function ec = get.enable_storing(this)
      ec = this.static_flags('storing');
    end

    function this = set.enable_storing(this, nec)
      ec = this.static_flags('storing', nec);
    end

    function dd = restore_detailed_data(cp, model, cp_index)
      if nargin <= 2
        cp_index = 'latest';
      end
      fp = Greedy.Checkpoint.filepath(model, cp.ids{1});
      cpfn = Greedy.Checkpoint.filename(model, cp.ids{1}, cp_index);
      tmp = load(fullfile(fp, cpfn));
      if length(cp.ids) == 1
        dd = tmp.detailed_data;
      else
        fp  = Greedy.Checkpoint.filepath(model, cp.ids{1});
        tmp = load(fullfile(fp, cp.filenames{1}));
        dd  = tmp.detailed_data;
        for i = 2:length(cp.ids)
          if ~isempty(cp.ids{i})
            fp  = Greedy.Checkpoint.filepath(model, cp.ids{i});
            tmp = load(fullfile(fp, cp.filenames{i}));
            set(dd, cp.data_index{i}, tmp.detailed_data);
          end
        end
      end
    end

    function ret = get(this, field, default)
      % function ret = get(this, field, default)
      % access function for user data stored in a Checkpoint instance
      %
      % Parameters:
      %   field: name of a field in the #userdata structure of this #level.
      %   default: This value is returned if the field does not exist in the
      %            #userdata structure.
      %
      % Return values:
      %   ret:  This returns the value of the userdata field.

      ud = this.userdata{this.level};
      if isfield(ud, field)
        ret = ud.(field);
      else
        ret = default;
      end
    end

    function child = child(this, index)
      % function child = child(this, index)
      % is called when a new Checkpoint instance for the child of a DataTree.INode
      % instance shall be generated.
      %
      % The child must be a DataTree.INode itself.
      %
      % Parameters:
      %   index: a scalar indicating the father child-relationship. child must
      %          be the 'index'-th child of father.
      %
      % Return values:
      %   child: the created instance of type Checkpoint
      child = Greedy.Checkpoint(this, index);
    end

%    function ret = subsref(this, S)
%      if isequal(S(1).type, '.') && isequal(S(1).subs, 'ud') ...
%          && length(S) == 2 && isequal(S(2).type, '.')
%        if isfield(this.userdata, S(2).subs)
%          ret = this.userdata.(S(2).subs);
%        else
%          ret = [];
%        end
%      else
%        ret = builtin('subsref', this, S);
%      end
%    end

    function cp = store(cp, model, detailed_data, id, ud)
      % function cp = store(cp, model, detailed_data, id, ud)
      % stores a DataTree.INode tree on the harddrive and attaches a descriptive
      % text and user data.
      %
      % The directory where the data is stored is computed via
      % Greedy.Checkpoint.filepath() from the model argument.
      %
      % @note The stored DataTree.INode knows its fathers, such that it can be
      %       embedded in a full DataTree.INode tree by restore().
      %
      % Parameters:
      %   model: an object or type IReducedModel
      %   detailed_data: an object of type DataTree.INode to be stored on the harddrive
      %   id:    id strings which describes the DataTree.INode instance or the
      %          algorithm which generates it.
      %   ud:    arbitrary user data to stored with this checkpoint.
      %
      % Return values:
      %   cp:    handle object of type Greedy.Checkpoint pointing to the changed
      %          Checkpoint instance.

      if cp.enable_storing
        fp = Greedy.Checkpoint.filepath(model, id);
        if ~exist(fp, 'dir')
          mkdir(fp);
        end

        files = dir(fp);

        dates = zeros(length(files),1);
        if length(files) >= cp.file_rotation_size + 2
          for i = 1:length(files)
            if isempty(setdiff(files(i).name, {'.', '..'}))
              dates(i) = inf;
            else
              dates(i) = files(i).datenum;
            end
          end
          [dates, ind] = min(dates);
          cpfn = files(ind).name;
        else
          cpfn = ['cp', num2str(length(files)-2)];
        end

        cp.ids{cp.level}       = id;
        cp.userdata{cp.level}  = structcpy(cp.userdata{cp.level}, ud);
        cp.filenames{cp.level} = cpfn;
        save(fullfile(fp, cpfn), 'detailed_data', 'cp');
      end
    end

  end

  methods (Static)

    function [dd, cp] = restore_latest_if_available(model, id)
      fp = Greedy.Checkpoint.filepath(model, id);
      if isempty(dir(fp)) || ~Greedy.Checkpoint.static_flags('auto_restore')
        dd = [];
        cp = Greedy.Checkpoint;
      else
        [dd, cp] = Greedy.Checkpoint.restore(model, id, 'latest');
      end
    end

    function [dd, cp] = restore(model, id, cp_index)
    % function [dd, cp] = restore(model, id[, cp_index])
    % restores a check point by a descriptive text and a checkpoint index
    %
    % Parameters:
    %   id: the descriptive id string used when storing the DataTree.INode with
    %       store().
    %   cp_index: an file index between '1' and #file_rotation_size or the
    %             string 'latest'. (Default = 'lastest')
    %
    % Return values:
    %   dd: restored object of type DataTree.INode . Note, that is the root of a
    %       data tree even if the 'id' argument refers to a child of it.
    %   cp: an object of type Greedy.Checkpoint restored together with the 'dd'
    %       object.

      if nargin <= 2
        cp_index = 'latest';
      end
      if nargin <= 1
        id = 'latest';
      end
      fp = Greedy.Checkpoint.filepath(model, id);
      if ~exist(fp, 'dir')
        error(['Could not find checkpoint! Directory ', fp, ' does not exist.']);
      end

      cpfn = Greedy.Checkpoint.filename(model, id, cp_index);

      tmp = load(fullfile(fp, cpfn));
      cp = tmp.cp;

      dd = restore_detailed_data(cp, model, cp_index);
      cp.level = 1;
    end

    function cpfn = filename(model, id, cp_index)
      fp = Greedy.Checkpoint.filepath(model, id);
      
      files = dir(fp);
      
      if isequal(cp_index, 'latest')
        dates = zeros(length(files),1);
        for i = 1:length(files)
          if isempty(setdiff(files(i).name, {'.', '..'}))
            dates(i) = 0;
          else
            dates(i) = datenum(files(i).date);
          end
        end
        [dates, ind] = max(dates);
        cpfn = files(ind).name;
      else
        cpfn = ['cp', num2str(cp_index)];
      end
    end

    function clear_old_checkpoints(model, num_left)
      % function clear_old_checkpoints(model [, num_left])
      % clears data files in a checkpoint directory
      %
      % The filepath is generated from the argument 'model'.
      %
      % Parameters:
      %   model:  an object of type IReducedModel .
      %   num_left: number of files to be left in the directory. (Default = 0)

      if nargin == 1
        num_left = 0;
      end

      fp = Greedy.Checkpoint.basepath;
      descr = model.descr;
      base = fullfile(fp, descr.name);
      if exist(base, 'dir')

        dirs = dir(base);
        dates = zeros(length(dirs),1);
        for i = 1:length(dirs)
          if isempty(setdiff(dirs(i).name, {'.', '..'}))
            dates(i) = inf;
          else
            dates(i) = datenum(dirs(i).date);
          end
        end
        [dates, inds] = sort(dates, 1, 'descend');

        for i = inds(num_left+1:end)
          rmdir(fullfile(base, dirs(i).name), 's')
        end
      end
    end

    function fp = basepath()
      % function fp = basepath()
      % returns the basepath for storage of Greedy.Checkpoint instances on the hdd.
      %
      % Return values:
      %   fp: string of file path to base directory.
      fp = fullfile(rbmatlabtemp, 'checkpoints');
    end

    function fp = filepath(rmodel, id)
      % function fp = filepath(rmodel, id)
      % computes a file path from the rmodel hash and an id
      %
      % Parameters:
      %   rmodel: an object of type IReducedModel
      %   id:     id strings which describes the DataTree.INode instance or the
      %           algorithm which generates it.
      %
      % Return values:
      %   fp: string of file path to the directory where data is stored.

      descr = rmodel.detailed_model;
      mhash = Greedy.Checkpoint.compute_hash(descr);
      fp = fullfile(Greedy.Checkpoint.basepath, descr.name, mhash, id);
    end

    function ids = get_ids(rmodel)
      % function ids = get_ids(rmodel)
      % returns a list of possible 'ids' to restore for a reduced model.
      %
      % Parameters:
      %   rmodel: an object of type IReducedModel
      %
      % Return values:
      %   ids: a cell array of possible ids to restore.

      ids = {};

      fp = Greedy.Checkpoint.filepath(rmodel, '');
      if exist(fp, 'dir')
        dirlist = dir(fp);
        ids = setdiff(arrayfun(@(x) x.name, dirlist, 'UniformOutput', false), {'.', '..'});
      end
    end

    function mhash = compute_hash(descr)
      % function mhash = compute_hash(descr)
      % computes an md5 hash string from the model description used to identify
      % it over a file name.
      %
      % Parameters:
      %   descr: a structure describing the model.
      %
      % Return values:
      %   mhash: a string of the hash key computed.
      il = [Greedy.Checkpoint.ignorelist, descr.mu_names];
      if isfield(descr, 'filecache_ignore_fields_in_model') || isprop(descr, 'filecache_ignore_fields_in_model')
        il = [ il, descr.filecache_ignore_fields_in_model ];
      end
      fns = setdiff(fieldnames(descr), il);
      fns = sort(fns);
      mstr = '';
      for i = 1:length(fns)
        mstr = [ mstr, fns{i}, ':', Greedy.Checkpoint.stringify(descr.(fns{i})) ];
      end
      mhash = hash(mstr, 'md5');
    end

    function str = stringify(x)
      % function str = stringify(x)
      % helper function to make strings out of function_handle, cell and other objects.
      %
      % This function is utilized by the compute_hash() method.
      %
      % Parameters:
      %   x: an arbitrary object from which a string shall be computed.
      %
      % Return values:
      %   str: the string
      if isobject(x)
        str = class(x);
      else
        switch(class(x))
          case 'function_handle'
            str = func2str(x);
          case 'cell'
            str = mat2str(cell2mat(cellfun(@Greedy.Checkpoint.stringify, x, 'UniformOutput', false) ) );
          case 'struct'
            str = 'structs_not_handled_yet';
          otherwise % double, logical, uint??, int??, ...
            str = mat2str(x);
        end
      end
      str = [str, ','];
    end
  end

  methods (Static, Access = public)

    function ec = static_flags(flag, nec)
      persistent pec;
      if isempty(pec)
        pec = struct('auto_restore', true, 'storing', true);
      end
      if nargin == 2
        pec.(flag) = nec;
      end
      ec = pec.(flag);
    end
  end

end