% Version 1.000
%
% Code provided by Geoff Hinton and Ruslan Salakhutdinov
%
% Permission is granted for anyone to copy, use, modify, or distribute this
% program and accompanying programs and documents for any purpose, provided
% this copyright notice is retained and prominently displayed, along with
% a note saying that the original programs are available from our
% web page.
% The programs and documents are distributed without any warranty, express or
% implied. As the programs were written for research purposes only, they have
% not been tested to the degree that would be advisable in any important
% application. All use of these programs is entirely at the user's own risk.
% This program trains Restricted Boltzmann Machine in which
% visible, binary, stochastic pixels are connected to
% hidden, binary, stochastic feature detectors using symmetrically
% weighted connections. Learning is done with 1-step Contrastive Divergence.
% The program assumes that the following variables are set externally:
% maxepoch -- maximum number of epochs
% numhid -- number of hidden units
% batchdata -- the data that is divided into batches (numcases numdims numbatches)
% restart -- set to 1 if learning starts from beginning
epsilonw = 0.1; % Learning rate for weights
epsilonvb = 0.1; % Learning rate for biases of visible units
epsilonhb = 0.1; % Learning rate for biases of hidden units
weightcost = 0.0002;
initialmomentum = 0.5;
finalmomentum = 0.9;
[numcases numdims numbatches]=size(batchdata); %%100x784x600
if restart ==1,
restart=0;
epoch=1;
% Initializing symmetric weights and biases.
vishid = 0.1*randn(numdims, numhid); %784x1000
hidbiases = zeros(1,numhid); % 1x1000
visbiases = zeros(1,numdims); % 1x784
poshidprobs = zeros(numcases,numhid); %100x1000
neghidprobs = zeros(numcases,numhid); %100x1000
posprods = zeros(numdims,numhid); %784x1000
negprods = zeros(numdims,numhid); %784x1000
vishidinc = zeros(numdims,numhid); %784x1000
hidbiasinc = zeros(1,numhid); %1x1000
visbiasinc = zeros(1,numdims); %1x784
batchposhidprobs=zeros(numcases,numhid,numbatches); %100x1000x600
end
for epoch = epoch:maxepoch, % 1 : 10
fprintf(1,'epoch %d\n',epoch);
errsum=0;
for batch = 1:numbatches,
fprintf(1,'epoch %d batch %d\n',epoch,batch);
%%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
data = batchdata(:,:,batch); % 100x784
poshidprobs = 1./(1 + exp(-data*vishid - repmat(hidbiases,numcases,1))); %100x784 * 784x1000 - 1x1000repmat->100x1000 = 100x1000
batchposhidprobs(:,:,batch)=poshidprobs; %100x1000<-batch b="" nbsp="">
posprods = data' * poshidprobs; %784x100 * 100x1000 = 784x1000
poshidact = sum(poshidprobs); % 1x1000
posvisact = sum(data); % 1x784
%%%%%%%%% END OF POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
poshidstates = poshidprobs > rand(numcases,numhid); %100x1000 > 100x1000 = 100 x 1000
%%%%%%%%% START NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
negdata = 1./(1 + exp(-poshidstates*vishid' - repmat(visbiases,numcases,1))); % 100x1000 * 1000x784 - 1x784repmat->100x784 = 100 x 784
neghidprobs = 1./(1 + exp(-negdata*vishid - repmat(hidbiases,numcases,1))); % 100x784 * 784x1000 - 1x1000repmat->100x1000 = 100x1000
negprods = negdata'*neghidprobs; % 784x100 * 100x1000
neghidact = sum(neghidprobs); % 1x1000
negvisact = sum(negdata); % 1x784
keyboard()
%%%%%%%%% END OF NEGATIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
err= sum(sum( (data-negdata).^2 )); %1x1
errsum = err + errsum;
if epoch>5,
momentum=finalmomentum;
else
momentum=initialmomentum;
end;
%%%%%%%%% UPDATE WEIGHTS AND BIASES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
vishidinc = momentum*vishidinc + ... % 784x1000
epsilonw*( (posprods-negprods)/numcases - weightcost*vishid); % 784x 1000
visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); % 1x784 + 1x784 - 1x784
hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); % 1x1000 + 1x1000 - 1x1000
vishid = vishid + vishidinc; % 784x1000 + 784x1000
visbiases = visbiases + visbiasinc; % 1x784 + x784
hidbiases = hidbiases + hidbiasinc; % 1x1000 + 1x1000
%%%%%%%%%%%%%%%% END OF UPDATES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
end
fprintf(1, 'epoch %4i error %6.1f \n', epoch, errsum);
end;
