Commit 7a9222f9 authored by sarthou's avatar sarthou
Browse files

Substitution of onion peel with loading done, but the algorithm is still not working

parent e28f425f
......@@ -78,7 +78,6 @@ void InpaintingConverger::OnionPeelInitialisation(InpaintingDataSource * const i
iProfiler->AfterInitialisation();
}
// TODO On doit compléter l'initiatlisation par chargement avec les matrices de déplacement en .csv
void InpaintingConverger::LoadingInitialisation(InpaintingDataSource * const iDataSource,InpaintingProfiler * const iProfiler){
iProfiler->BeforeInitialisation();
InpaintingOpenCVIO openCVIO = InpaintingOpenCVIO();
......@@ -89,6 +88,9 @@ void InpaintingConverger::LoadingInitialisation(InpaintingDataSource * const iDa
PaddedMatrix<int> * const frame_x = new PaddedMatrix<int>(iDataSource->GetDim());
PaddedMatrix<int> * const frame_y = new PaddedMatrix<int>(iDataSource->GetDim());
PaddedMatrix<int> * const frame_z = new PaddedMatrix<int>(iDataSource->GetDim());
PaddedMatrix<float> * const frame_a = new PaddedMatrix<float>(iDataSource->GetDim());
PaddedMatrix<float> * const frame_gx = new PaddedMatrix<float>(iDataSource->GetDim());
PaddedMatrix<float> * const frame_gy = new PaddedMatrix<float>(iDataSource->GetDim());
for(int nbFrame=0;nbFrame<iDataSource->GetDim()->GetSizeZ();nbFrame++)
{
......@@ -103,6 +105,19 @@ void InpaintingConverger::LoadingInitialisation(InpaintingDataSource * const iDa
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_DispZ_"\
+std::to_string(nbFrame)+".csv",frame_z,1);
iDataSource->GetDispZ()->SetFrame(frame_z,nbFrame);
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_DispA_"\
+std::to_string(nbFrame)+".csv",frame_a,1);
iDataSource->GetDispA()->SetFrame(frame_a,nbFrame);
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_GradX_"\
+std::to_string(nbFrame)+".csv",frame_gx,1);
iDataSource->GetGradX()->SetFrame(frame_gx,nbFrame);
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_GradY_"\
+std::to_string(nbFrame)+".csv",frame_gy,1);
iDataSource->GetGradY()->SetFrame(frame_gy,nbFrame);
}
//iDataSource->GetDispX()->GetFrame(27)->Mprint();
......
......@@ -23,7 +23,7 @@ int main(){
// Select and parameterize the algorithms
InpaintingEvaluator evaluationAlgo = InpaintingEvaluator(50, 5, 5, 5);
ISRandom searchAlgo = ISRandom(20, &evaluationAlgo); //initiated at 20, but 10 in Matlab code
ISRandom searchAlgo = ISRandom(30, &evaluationAlgo); //initiated at 20, but 10 in Matlab code
// ISFull searchAlgo = ISFull(&evaluationAlgo);
IRWeightedAverage reconstructAlgo = IRWeightedAverage(0.75f, 5, 5, 5);
//IRBestPatch reconstructAlgo = IRBestPatch(5, 5, 5);
......@@ -76,8 +76,8 @@ int main(){
// Output the new images.
for(int i = 0; i < baseIPS.GetDim()->GetSizeZ(); ++i)
openCVIO.GeneralIOForRGB(baseIPS.GetRed(), baseIPS.GetGreen(), baseIPS.GetBlue(), i,
InpaintingIO::IH_DBG_SAVE, "Data/onion_debug/CppOut_WithMatlabInput_F" + std::to_string(i));
InpaintingIO::IH_DBG_SAVE, "Data/onion_debug/CppOut_WithMatlabInput30_F" + std::to_string(i));
profiler.AfterInpainting();
return 0;
......
......@@ -285,19 +285,25 @@ function[imgOut, shiftVolOut] = inpaint_video(varargin)
debug_iter = debug_iter +1;
end
% end of initialisation branch
%write_image_volume(imgVol,'matlab_debug/peel/M_EndPeel_F');
cd matlab_debug/peel/end_peel_disp/
[~,w,l,maxFrame] = size(shiftVol);
for nbframe=1:maxFrame
csvwrite(strcat('M_EndPeel_DispX_',int2str(nbframe-1),'.csv'),reshape(shiftVol(1,:,:,nbframe),[w,l]));
csvwrite(strcat('M_EndPeel_DispY_',int2str(nbframe-1),'.csv'),reshape(shiftVol(2,:,:,nbframe),[w,l]));
csvwrite(strcat('M_EndPeel_DispZ_',int2str(nbframe-1),'.csv'),reshape(shiftVol(3,:,:,nbframe),[w,l]));
end
cd ../../..
% cd matlab_debug/peel/end_peel_disp/
%
% [~,w,l,maxFrame] = size(shiftVol);
%
% for nbframe=1:maxFrame
% csvwrite(strcat('M_EndPeel_DispX_',int2str(nbframe-1),'.csv'),reshape(shiftVol(1,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_EndPeel_DispY_',int2str(nbframe-1),'.csv'),reshape(shiftVol(2,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_EndPeel_DispZ_',int2str(nbframe-1),'.csv'),reshape(shiftVol(3,:,:,nbframe),[w,l]));
%
% csvwrite(strcat('M_EndPeel_DispA_',int2str(nbframe-1),'.csv'),reshape(shiftVol(4,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_EndPeel_GradX_',int2str(nbframe-1),'.csv'),normGradX(:,:,nbframe));
% csvwrite(strcat('M_EndPeel_GradY_',int2str(nbframe-1),'.csv'),normGradY(:,:,nbframe));
%
%
% end
%
% cd ../../..
end
pp=pp+1;
......
%this function inpaints a video
% inputs:
% 1/ input video
% 2/ occlusion video
% 3/ inpainting parameters
function[imgOut, shiftVolOut] = inpaint_video(varargin)
imgVolIn = varargin{1};
occVolIn = varargin{2};
%FIXED PARMETERS !!!
%GAUSSIAN PYRAMID PARAMETERS
filterSize = 3; %fixed
scaleStep = 0.5; %fixed
sigma = 1.5; %fixed
useAllPatches = 0;
reconstructionType = 0;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%parse inpainting parameters
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
[maxLevel,patchSizeX,patchSizeY,patchSizeT,textureFeaturesActivated,sigmaColour,file] = ...
parse_inpaint_parameters(varargin{3});
patchSize = [patchSizeX patchSizeY min(patchSizeT,size(imgVolIn,4))];
patchMatchParams.patchSize = [patchSizeX patchSizeY min(patchSizeT,size(imgVolIn,4))];
patchMatchParams.patchSizeX = patchSize(1);
patchMatchParams.patchSizeY = patchSize(2);
patchMatchParams.patchSizeT = patchSize(3);
patchMatchParams.w = max([size(imgVolIn,2) size(imgVolIn,3) size(imgVolIn,4)]); %manual
patchMatchParams.alpha = 0.5; %fixed
patchMatchParams.fullSearch = 0;
patchMatchParams.partialComparison = 1;
patchMatchParams.nbItersPatchMatch = 10;
patchMatchParams.patchIndexing = 0;
patchMatchParams.reconstructionType = reconstructionType;
%parameters concerning the iterations of the nearest neighbour search
%and reconstruction
maxNbIterations = 20;
residualThresh = 0.1;
%%%% get pyramid volumes
imgVolPyramid = get_image_volume_pyramid(imgVolIn,filterSize,sigma,maxLevel,size(imgVolIn,4));%patchSize(3)
occVolPyramid = get_image_volume_pyramid(occVolIn,filterSize,sigma,maxLevel,size(imgVolIn,4));%patchSize(3)
%%%%% create texture feature pyramid
if (textureFeaturesActivated>0)
disp('Calculating texture feature pyramids');
featurePyramid = get_video_features(imgVolIn,occVolIn,maxLevel,file);
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% iterate over all pyramid levels
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
for ii=maxLevel:-1:1
pp=1;
residual = inf;
occVol = single(occVolPyramid{ii});
imgVol(1,:,:,:) = imgVolPyramid{ii,1}; %get the red channel for this image volume
imgVol(2,:,:,:) = imgVolPyramid{ii,2}; %get the blue channel for this image volume
imgVol(3,:,:,:) = imgVolPyramid{ii,3}; %get the green channel for this image volume
if (exist('featurePyramid','var'))
gradX = single(featurePyramid{ii,1});
gradY = single(featurePyramid{ii,2});
normGradX = single(featurePyramid{ii,3});
normGradY = single(featurePyramid{ii,4});
patchMatchParams.gradX = gradX;
patchMatchParams.gradY = gradY;
patchMatchParams.normGradX = normGradX;
patchMatchParams.normGradY = normGradY;
end
% If we are not at the coarsest level...
if (ii~=maxLevel)
% ... upsample the previous shift map...
shiftVol = single(interpolate_disp_field(shiftVol,imgVol,1/scaleStep, patchSize,'nearest'));
% ... and recreate image volume.
if (exist('featurePyramid','var'))
[imgVol,gradX,gradY,normGradX,normGradY] = ...
reconstruct_video_and_features_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
patchMatchParams.gradX = single(gradX);
patchMatchParams.gradY = single(gradY);
patchMatchParams.normGradX = single(normGradX);
patchMatchParams.normGradY = single(normGradY);
else
[imgVol] = reconstruct_video_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
end
end
% Dilate the occlusion
structElPatch = strel('arbitrary', ones(patchSize(2),patchSize(1),patchSize(3)));
occVolDilate = imdilate(occVol,structElPatch);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% start of iterative inpainting at this level
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
while (pp <= maxNbIterations && residual > residualThresh)
imgVolIterMinusOne = imgVol+1-1;
if (ii ~= maxLevel || pp>1) %not bottom level
patchMatchParams.partialComp = 0;
patchMatchParams.partialComparison = 0;
useAllPatches = 1;
% Copy last shiftVol
firstGuess = shiftVol+1-1;
% Carry out the 3D PatchMatch
shiftVol = spatio_temporal_patch_match_mex(...
imgVol,...
imgVol,...
patchMatchParams,...
firstGuess,...
occVolDilate,...
occVolDilate);
% endfile = strcat('_',strcat(int2str(100*(maxLevel-ii+1)+pp-1),'.csv'));
%
% cd matlab_debug
%
% [~,w,l,maxFrame] = size(shiftVol);
%
% for nbframe=1:maxFrame
% csvwrite(strcat('M_DispX_',strcat(int2str(nbframe),endfile)),reshape(shiftVol(1,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_DispY_',strcat(int2str(nbframe),endfile)),reshape(shiftVol(2,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_DispZ_',strcat(int2str(nbframe),endfile)),reshape(shiftVol(3,:,:,nbframe),[w,l]));
% end
%
% cd ..
% Carry out the reconstruction
if (exist('featurePyramid','var'))
[imgVol,gradX,gradY,normGradX,normGradY] = ...
reconstruct_video_and_features_mex(...
imgVol,...
occVol,...
shiftVol,patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
patchMatchParams.gradX = single(gradX);
patchMatchParams.gradY = single(gradY);
patchMatchParams.normGradX = single(normGradX);
patchMatchParams.normGradY = single(normGradY);
else
[imgVol] = reconstruct_video_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
end
else
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% INITIALISATION %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
occVolIter = occVol;
debug_iter =0;
%fill in, in an onion peel fashion
while(sum(occVolIter(:)) > 0)
structElCube = strel('arbitrary', ones(3,3,3));
occVolErode = imerode(occVolIter,structElCube);
%set up the partial occlusion volume for the PatchMatch :
% - 0 for non-occlusion;
% - 1 for occluded and not to take into account when
% comparing patches
% - 2 for occluded and to take into account (we do not allow
%the nearest neighbours to point to these pixels, but
%they have been reconstructed at this iteration)
occVolPatchMatch = occVolDilate;
occVolPatchMatch((occVolDilate - occVolIter) == 1) = 2;
%initial guess : by default, set everything to 0
if (~exist('shiftVol','var'))
firstGuess = single(zeros([4 size(imgVol,2) size(imgVol,3) size(imgVol,4)]));
else
firstGuess = shiftVol+1-1;
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%carry out the 3D patchMatch
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
shiftVol = ...
spatio_temporal_patch_match_mex(...
imgVol,...
imgVol,...
patchMatchParams,...
firstGuess,...
occVolPatchMatch,...
occVolDilate);
%determine the pixels to reconstruct at this layer
occVolBorder = abs(occVolIter - occVolErode);
%if the occVol == 2, then we cannot use the colour
%info, but we do not inpaint it either
occVolBorder(occVolErode == 1) = 2;
cd matlab_debug
[~,w,l,maxFrame] = size(shiftVol);
write_image_volume(O
for nbframe=1:maxFrame
csvwrite(strcat('M_Init_DispX_',strcat(int2str(nbframe),'.csv')),reshape(shiftVol(1,:,:,nbframe),[w,l]));
csvwrite(strcat('M_Init_DispY_',strcat(int2str(nbframe),'.csv')),reshape(shiftVol(2,:,:,nbframe),[w,l]));
csvwrite(strcat('M_Init_DispZ_',strcat(int2str(nbframe),'.csv')),reshape(shiftVol(3,:,:,nbframe),[w,l]));
end
cd ..
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%carry out the reconstruction
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if (exist('featurePyramid','var') && exist('shiftVol','var'))
[imgVol,gradX,gradY,normGradX,normGradY] = ...
reconstruct_video_and_features_mex(...
imgVol,...
occVolBorder,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
patchMatchParams.gradX = single(gradX);
patchMatchParams.gradY = single(gradY);
patchMatchParams.normGradX = single(normGradX);
patchMatchParams.normGradY = single(normGradY);
else
[imgVol] = reconstruct_video_mex(...
imgVol,...
occVolBorder,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
end
%now go to the next onion peel layer
occVolIter = occVolErode;
debug_iter = debug_iter +1;
end
end
pp=pp+1;
%we have finished the initialisation : make sure that both
%the patchMatch AND the reconstruction consider that
%everything is known (no more partial patch comparisons)
patchMatchParams.partialComparison = 0;
patchMatchParams.useAllPatches = 1;
useAllPatches = 1;
%calculate the residual to see if we terminate
residual = sum(abs(imgVolIterMinusOne(:) - imgVol(:)))/(single(3*sum(occVol(:)>0)));
end
if (ii==1)
imgVol = reconstruct_video_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,1);
% occInds = find(occVol>0);
% energy = sum(shiftVol(occInds + 3*prod(sizeImgVol(1:2))));
imgOut = imgVol;
shiftVolOut = shiftVol;
size(shiftVolOut)
% cd ../Data;
% write_image_volume(shiftVolOut,strcat('disp',str((maxLevel-ii)*100 + pp);
% cd ../MATLAB Inpainting
return;
end
%erase some of the video structures
clearvars imgVol occVolDilate occVolPatchMatch;
patchMatchParams.partialComparison = 0;
useAllPatches = 1;
end
imgOut = imgVol;
end
......@@ -19,6 +19,7 @@ Cpp30 = "Data/onion_debug/CppOut30F"
PeelMatlab = "Data/onion_debug/peel_compare/M_EndPeel_F_frame_"
PeelCpp = "Data/onion_debug/peel_compare/Cpp_EndPeel_F"
OccSmallPath = "Data/onion_debug/Occ/OccF0.png"
CppLoadingPath = "Data/onion_debug/CppOut_WithMatlabInput_F"
# print(OccMask)
threshold_nbPixDiff = 0.1
......@@ -178,6 +179,7 @@ if __name__ == '__main__':
VolCpp30 = np.zeros((98,68,264,3))
VolPeelM = np.zeros((98,17,66,3))
VolPeelC = np.zeros((98,17,66,3))
VolCppLoad = np.zeros((98,68,264,3))
# Chargement des images dans un volume
for i in range(98):
if(i<9):
......@@ -194,6 +196,7 @@ if __name__ == '__main__':
# img_cpp30 = io.imread(Cpp30+str(i)+".png")
img_peelM = io.imread(PeelMatlab+str(i)+".png")
img_peelC = io.imread(PeelCpp+str(i)+".png")
img_load = io.imread(CppLoadingPath+str(i)+".png")
# print(img_cpp.shape)
......@@ -206,7 +209,7 @@ if __name__ == '__main__':
# VolCpp30[i,:,:,:]=img_cpp30
VolPeelC[i,:,:,:]=img_peelC
VolPeelM[i,:,:,:]=img_peelM
VolCppLoad[i,:,:,:]=img_load
# # Code for video
# cap = cv2.VideoCapture(OriFilePath)
# i=0
......@@ -241,17 +244,33 @@ if __name__ == '__main__':
# # Affichage résultat de la comparaison des pixels dans la zone du masque
# pixCountRGB = np.zeros((3,98))
# for i in range(98):
# pixCountRGB[:,i]=PixCompare(VolPeelM[i,:,:,:],VolPeelC[i,:,:,:],OccSmallMask,0.1)
# pixCountRGB[:,i]=PixCompare(VolCpp[i,:,:,:],VolCppLoad[i,:,:,:],OccMask,0.1)
#
# plt.plot(pixCountRGB[0,:],'r', label='Red pixel ratio')
# plt.plot(pixCountRGB[1,:],'g',label='Green pixel ratio')
# plt.plot(pixCountRGB[2,:],'b',label='Blue pixel ratio')
# plt.plot((pixCountRGB[0,:]+pixCountRGB[1,:]+pixCountRGB[2,:])/3,'black',label='Mean pixel ratio')
# plt.axhline(0.05)
# # plt.axhline(0.05)
# plt.xlabel("number of frame (time)")
# plt.ylabel("Percentage of pixels different in each image on the mask")
# plt.legend()
# plt.title("Evolution de la différence de pixel entre le code C++ et Matlab à la fin de l'initialisation en pelure d'onion")
# plt.title("Evolution de la différence de pixel entre le code C++ avec et sans initialisation en pelure d'onion")
# plt.show()
# # Affichage résultat de la comparaison des pixels dans la zone du masque
# pixCountCpp = np.zeros((3,98))
# pixCountLoad = np.zeros((3,98))
# for i in range(98):
# pixCountCpp[:,i]=PixCompare(VolCpp[i,:,:,:],VolMatlab[i,:,:,:],OccMask,0.1)
# pixCountLoad[:,i]=PixCompare(VolCppLoad[i,:,:,:],VolMatlab[i,:,:,:],OccMask,0.1)
#
# plt.plot((pixCountCpp[0,:]+pixCountCpp[1,:]+pixCountCpp[2,:])/3,'black',label='Cpp with peel init')
# plt.plot((pixCountLoad[0,:]+pixCountLoad[1,:]+pixCountLoad[2,:])/3,'blue',label='Cpp with matlab data loading')
# # plt.axhline(0.05)
# plt.xlabel("number of frame (time)")
# plt.ylabel("Percentage of pixels different in each image on the mask")
# plt.legend()
# plt.title("Evolution de la différence de pixel entre le code C++ avec et sans initialisation en pelure d'onion")
# plt.show()
# ## Evolution du PSNR dans la vidéo
......@@ -283,6 +302,19 @@ if __name__ == '__main__':
# plt.title("Comparaison of mean PSNR over RGB during time for differents search iterations in the C++ algorithm")
# plt.show()
## PSNR entre Cpp avec init_peel ou en chargeant les données matlab (reference avec Matlab)
# PSNR_vol = np.zeros((2,98))
# for i in range(98):
# [_,_,_,PSNR_vol[0,i]] = PSNR(VolMatlab[i,:,:,:],VolCpp[i,:,:,:],OccMask)
# [_,_,_,PSNR_vol[1,i]] = PSNR(VolMatlab[i,:,:,:],VolCppLoad[i,:,:,:],OccMask)
# plt.plot(PSNR_vol[0,:],'r', label='Cpp with onion peel init')
# plt.plot(PSNR_vol[1,:],'g',label='Cpp with loading matlab data')
# plt.ylabel("PSNR on the mask during time")
# plt.xlabel("number of frame (time)")
# plt.legend()
# plt.title("Comparaison of mean PSNR over RGB during time for C++ algorithm with or without self onion peel initialisation")
# plt.show()
# ## Evolution du SSIM dans la vidéo en comparaison CPP 10/20/30 search iter
# SSIM_cpp10 = np.zeros((3,98))
# SSIM_cpp =np.zeros((3,98))
......@@ -325,4 +357,4 @@ if __name__ == '__main__':
# plt.show()
# VolOutComp = visual_compare(VolRand1,VolRand2,"Data/random_compare/RComp")
visual_compare(VolPeelM,VolPeelC,"Data/onion_debug/peel_compare/CompF")
visual_compare(VolCpp,VolCppLoad,"Data/onion_debug/peel_compare/visual_comp_cpp_matlab_loading/CompLoadF")
......@@ -84,6 +84,6 @@ p1 = Image.fromarray(np.uint8(p_temp))
for i in range(98):
p1 = Image.open("Data/Out_matlab_random/beach_umbrellainpainted_frame_0"+str(i)+".png")
p2 = Image.open("Data/onion_debug/CppOutF"+str(i)+".png")
p2 = Image.open("Data/onion_debug/CppOut_WithMatlabInput_F"+str(i)+".png")
p3 = Image.blend(p1,p2,.8)
p3.save("Data/onion_debug/WTF_"+str(i)+".png")
p3.save("Data/onion_debug/Merge_"+str(i)+".png")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment