% Version 1.000
%
% Code provided by Ruslan Salakhutdinov and Geoff Hinton
%
% Permission is granted for anyone to copy, use, modify, or distribute this
% program and accompanying programs and documents for any purpose, provided
% this copyright notice is retained and prominently displayed, along with
% a note saying that the original programs are available from our
% web page.
% The programs and documents are distributed without any warranty, express or
% implied. As the programs were written for research purposes only, they have
% not been tested to the degree that would be advisable in any important
% application. All use of these programs is entirely at the user's own risk.
% This program trains Restricted Boltzmann Machine in which
% visible, binary, stochastic pixels are connected to
% hidden, stochastic real-valued feature detectors drawn from a unit
% variance Gaussian whose mean is determined by the input from
% the logistic visible units. Learning is done with 1-step Contrastive Divergence.
% The program assumes that the following variables are set externally:
% maxepoch -- maximum number of epochs
% numhid -- number of hidden units
% batchdata -- the data that is divided into batches (numcases numdims numbatches)
% restart -- set to 1 if learning starts from beginning
epsilonw = 0.001; % Learning rate for weights
epsilonvb = 0.001; % Learning rate for biases of visible units
epsilonhb = 0.001; % Learning rate for biases of hidden units
weightcost = 0.0002;
initialmomentum = 0.5;
finalmomentum = 0.9;
[numcases numdims numbatches]=size(batchdata); % 100x250x600
if restart ==1,
restart=0;
epoch=1;
% Initializing symmetric weights and biases.
vishid = 0.1*randn(numdims, numhid); % 250x30 = 250,30
hidbiases = zeros(1,numhid); % 1x30
visbiases = zeros(1,numdims); % 1x250
poshidprobs = zeros(numcases,numhid); % 100x30
neghidprobs = zeros(numcases,numhid); % 100x30
posprods = zeros(numdims,numhid); % 250x30
negprods = zeros(numdims,numhid); % 250x30
vishidinc = zeros(numdims,numhid); % 250x30
hidbiasinc = zeros(1,numhid); % 1x30
visbiasinc = zeros(1,numdims); % 1x250
sigmainc = zeros(1,numhid); % 1x30
batchposhidprobs=zeros(numcases,numhid,numbatches); % 100x30x600
end
for epoch = epoch:maxepoch,
fprintf(1,'epoch %d\n',epoch);
errsum=0;
for batch = 1:numbatches, % 1:600
fprintf(1,'epoch %d batch %d\n',epoch,batch);
%%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
data = batchdata(:,:,batch); %100x250 x600
poshidprobs = (data*vishid) + repmat(hidbiases,numcases,1); % 100x250 * 250x30 + 1x30repmat 100-> 100x30 = 100x30
batchposhidprobs(:,:,batch)=poshidprobs; %100x30
posprods = data' * poshidprobs; % 250x100 * 100x30 = 250x30
poshidact = sum(poshidprobs); % 1x30
posvisact = sum(data); % 1x250
%%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
poshidstates = poshidprobs+randn(numcases,numhid); % 100x30
%%%%%%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
negdata = 1./(1 + exp(-poshidstates*vishid' - repmat(visbiases,numcases,1)));
neghidprobs = (negdata*vishid) + repmat(hidbiases,numcases,1);
negprods = negdata'*neghidprobs;
neghidact = sum(neghidprobs);
negvisact = sum(negdata);
keyboard()
%%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
err= sum(sum( (data-negdata).^2 ));
errsum = err + errsum;
if epoch>5,
momentum=finalmomentum;
else
momentum=initialmomentum;
end;
%%%%%%%%% UPDATE WEIGHTS AND BIASES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
vishidinc = momentum*vishidinc + ... % 250x30
epsilonw*( (posprods-negprods)/numcases - weightcost*vishid); % 250x30
visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); % 1x250
hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); % 1x30
vishid = vishid + vishidinc; % 250x30
visbiases = visbiases + visbiasinc; % 1x250
hidbiases = hidbiases + hidbiasinc; % 1x30
%%%%%%%%%%%%%%%% END OF UPDATES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
end
fprintf(1, 'epoch %4i error %f \n', epoch, errsum);
end
