function avRew = evalPOMDPpolicy(pomdpFileName, policyFileName, nRuns, nSteps, initStateConfig)
% function avRew = evalPOMDPpolicy(pomdpFileName, policyFileName, nRuns, nSteps, initStateConfig)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Author: Pascal Poupart (Copyright 2007)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Inputs:
%
% pomdpFileName: file containing the POMDP description
% policyFileName: file containing the policy
% nRuns: number of policy runs (default value: 500)
% nSteps: number of time steps in each run (default value: 50)
% initStateConfig: initial state (default value: initial belief state)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Outputs:
%
% avRew: expected total discounted reward averaged over all runs
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Comments:
%
% Computes the expected total discounted reward averaged over all runs.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% load POMDP problem
fprintf('Loading policy...\n');
POMDP = ParseSPUDD(pomdpFileName);
POMDP.parsePOMDP(false);
ddPOMDP = dd_pomdp_flatAct(POMDP);

stateVars = 1:ddPOMDP.nStateVars;
observedVars = stateVars;
stateVarsPrime = stateVars+ddPOMDP.nVars;
obsVars = ddPOMDP.nStateVars+1:ddPOMDP.nVars;
obsVarsPrime = obsVars+ddPOMDP.nVars;

% load POMDP policy
load(policyFileName);
ddPOMDP = dd_pomdp_flatAct(POMDP);

if ~exist('nRuns')
  nRuns = 500;
end

if ~exist('nSteps');
  nSteps = 50;
end

avRew = 0;
for runId = 1:nRuns

  runId = runId
  rand('state',runId);
  Global.random.setSeed(runId);
  belState = ddPOMDP.initialBelState;
  if exist('initStateConfig','var')
    stateConfig = initStateConfig;
  else
    stateConfig = OP.sampleMultinomial(belState,stateVars);
  end
  
  totRew = 0;
  for stepId = 1:nSteps
    %stepId = stepId
    
    % pick action
    actId =  policyQuery(stateVars,belState,valueFunction,policy);
    %ddPOMDP.actions(actId).name

    % sum rewards
    %immRew =  OP.dotProduct(belState,ddPOMDP.actions(actId).rewFn, stateVars)
    totRew = totRew + ddPOMDP.discFact^(stepId-1) * OP.eval(ddPOMDP.actions(actId).rewFn, stateConfig);
    %if totRew > 217
    %  keyboard
    %end

    % sample observation
    restrictedTransFn = OP.restrictN(ddPOMDP.actions(actId).transFn, stateConfig); 
    stateConfig2 = OP.sampleMultinomial(restrictedTransFn,stateVarsPrime);
    restrictedObsFn = OP.restrictN(ddPOMDP.actions(actId).obsFn, [stateConfig, stateConfig2]);
    obsConfig = OP.sampleMultinomial(restrictedObsFn, obsVarsPrime);
    %Global.valNames(obsConfig(1,1),obsConfig(2,1))
    %Global.valNames(obsConfig(1,2),obsConfig(2,2))

    %keyboard
    
    % update belief state
    belState = beliefUpdate(ddPOMDP, belState, actId, obsConfig);
    stateConfig = Config.primeVars(stateConfig2,-ddPOMDP.nVars);
    %if stateConfig(2,2) == 2  % handswashed == true
    %  break;
    %end
    Global.newHashtables;
  end

  %OP.addMultVarElim(belState,[1:4,6,7])
  %OP.addMultVarElim(belState,[1:5,7])
  %OP.addMultVarElim(belState,[1:6])
  avRew = ((runId-1)*avRew + totRew) / runId;
  fprintf('totRew = %f\t avRew = %f\n',totRew,avRew);
end

