Skip to content
GitLab
Commit 7a9222f9 authored by sarthou's avatar sarthou
Browse files

Substitution of onion peel with loading done, but the algorithm is still not working

parent e28f425f
Show whitespace changes
Inline Side-by-side
InpaintingAlgorithm/algorithm/inpainting/InpaintingConverger.cpp
View file @ 7a9222f9
......@@ -78,7 +78,6 @@ void InpaintingConverger::OnionPeelInitialisation(InpaintingDataSource * const i
iProfiler->AfterInitialisation();
}
// TODO On doit compléter l'initiatlisation par chargement avec les matrices de déplacement en .csv
void InpaintingConverger::LoadingInitialisation(InpaintingDataSource * const iDataSource,InpaintingProfiler * const iProfiler){
iProfiler->BeforeInitialisation();
InpaintingOpenCVIO openCVIO = InpaintingOpenCVIO();
......@@ -89,6 +88,9 @@ void InpaintingConverger::LoadingInitialisation(InpaintingDataSource * const iDa
PaddedMatrix<int> * const frame_x = new PaddedMatrix<int>(iDataSource->GetDim());
PaddedMatrix<int> * const frame_y = new PaddedMatrix<int>(iDataSource->GetDim());
PaddedMatrix<int> * const frame_z = new PaddedMatrix<int>(iDataSource->GetDim());
PaddedMatrix<float> * const frame_a = new PaddedMatrix<float>(iDataSource->GetDim());
PaddedMatrix<float> * const frame_gx = new PaddedMatrix<float>(iDataSource->GetDim());
PaddedMatrix<float> * const frame_gy = new PaddedMatrix<float>(iDataSource->GetDim());
for(int nbFrame=0;nbFrame<iDataSource->GetDim()->GetSizeZ();nbFrame++)
{
......@@ -103,6 +105,19 @@ void InpaintingConverger::LoadingInitialisation(InpaintingDataSource * const iDa
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_DispZ_"\
+std::to_string(nbFrame)+".csv",frame_z,1);
iDataSource->GetDispZ()->SetFrame(frame_z,nbFrame);
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_DispA_"\
+std::to_string(nbFrame)+".csv",frame_a,1);
iDataSource->GetDispA()->SetFrame(frame_a,nbFrame);
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_GradX_"\
+std::to_string(nbFrame)+".csv",frame_gx,1);
iDataSource->GetGradX()->SetFrame(frame_gx,nbFrame);
openCVIO.LoadMatrixFromCSV("Data/onion_debug/peel_compare/end_peel_disp/M_EndPeel_GradY_"\
+std::to_string(nbFrame)+".csv",frame_gy,1);
iDataSource->GetGradY()->SetFrame(frame_gy,nbFrame);
}
//iDataSource->GetDispX()->GetFrame(27)->Mprint();
......
InpaintingAlgorithm/main.cpp
View file @ 7a9222f9
......@@ -23,7 +23,7 @@ int main(){
// Select and parameterize the algorithms
InpaintingEvaluator evaluationAlgo = InpaintingEvaluator(50, 5, 5, 5);
ISRandom searchAlgo = ISRandom(20, &evaluationAlgo); //initiated at 20, but 10 in Matlab code
ISRandom searchAlgo = ISRandom(30, &evaluationAlgo); //initiated at 20, but 10 in Matlab code
// ISFull searchAlgo = ISFull(&evaluationAlgo);
IRWeightedAverage reconstructAlgo = IRWeightedAverage(0.75f, 5, 5, 5);
//IRBestPatch reconstructAlgo = IRBestPatch(5, 5, 5);
......@@ -76,7 +76,7 @@ int main(){
// Output the new images.
for(int i = 0; i < baseIPS.GetDim()->GetSizeZ(); ++i)
openCVIO.GeneralIOForRGB(baseIPS.GetRed(), baseIPS.GetGreen(), baseIPS.GetBlue(), i,
InpaintingIO::IH_DBG_SAVE, "Data/onion_debug/CppOut_WithMatlabInput_F" + std::to_string(i));
InpaintingIO::IH_DBG_SAVE, "Data/onion_debug/CppOut_WithMatlabInput30_F" + std::to_string(i));
profiler.AfterInpainting();
......
MATLAB Inpainting/inpaint_video.m
View file @ 7a9222f9
......@@ -285,19 +285,25 @@ function[imgOut, shiftVolOut] = inpaint_video(varargin)
debug_iter = debug_iter +1;
end
% end of initialisation branch
%write_image_volume(imgVol,'matlab_debug/peel/M_EndPeel_F');
cd matlab_debug/peel/end_peel_disp/
[~,w,l,maxFrame] = size(shiftVol);
for nbframe=1:maxFrame
csvwrite(strcat('M_EndPeel_DispX_',int2str(nbframe-1),'.csv'),reshape(shiftVol(1,:,:,nbframe),[w,l]));
csvwrite(strcat('M_EndPeel_DispY_',int2str(nbframe-1),'.csv'),reshape(shiftVol(2,:,:,nbframe),[w,l]));
csvwrite(strcat('M_EndPeel_DispZ_',int2str(nbframe-1),'.csv'),reshape(shiftVol(3,:,:,nbframe),[w,l]));
end
cd ../../..
%write_image_volume(imgVol,'matlab_debug/peel/M_EndPeel_F');
% cd matlab_debug/peel/end_peel_disp/
%
% [~,w,l,maxFrame] = size(shiftVol);
%
% for nbframe=1:maxFrame
% csvwrite(strcat('M_EndPeel_DispX_',int2str(nbframe-1),'.csv'),reshape(shiftVol(1,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_EndPeel_DispY_',int2str(nbframe-1),'.csv'),reshape(shiftVol(2,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_EndPeel_DispZ_',int2str(nbframe-1),'.csv'),reshape(shiftVol(3,:,:,nbframe),[w,l]));
%
% csvwrite(strcat('M_EndPeel_DispA_',int2str(nbframe-1),'.csv'),reshape(shiftVol(4,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_EndPeel_GradX_',int2str(nbframe-1),'.csv'),normGradX(:,:,nbframe));
% csvwrite(strcat('M_EndPeel_GradY_',int2str(nbframe-1),'.csv'),normGradY(:,:,nbframe));
%
%
% end
%
% cd ../../..
end
pp=pp+1;
......
MATLAB Inpainting/inpaint_video.m~ deleted 100644 → 0
View file @ e28f425f
%this function inpaints a video
% inputs:
% 1/ input video
% 2/ occlusion video
% 3/ inpainting parameters
function[imgOut, shiftVolOut] = inpaint_video(varargin)
imgVolIn = varargin{1};
occVolIn = varargin{2};
%FIXED PARMETERS !!!
%GAUSSIAN PYRAMID PARAMETERS
filterSize = 3; %fixed
scaleStep = 0.5; %fixed
sigma = 1.5; %fixed
useAllPatches = 0;
reconstructionType = 0;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%parse inpainting parameters
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
[maxLevel,patchSizeX,patchSizeY,patchSizeT,textureFeaturesActivated,sigmaColour,file] = ...
parse_inpaint_parameters(varargin{3});
patchSize = [patchSizeX patchSizeY min(patchSizeT,size(imgVolIn,4))];
patchMatchParams.patchSize = [patchSizeX patchSizeY min(patchSizeT,size(imgVolIn,4))];
patchMatchParams.patchSizeX = patchSize(1);
patchMatchParams.patchSizeY = patchSize(2);
patchMatchParams.patchSizeT = patchSize(3);
patchMatchParams.w = max([size(imgVolIn,2) size(imgVolIn,3) size(imgVolIn,4)]); %manual
patchMatchParams.alpha = 0.5; %fixed
patchMatchParams.fullSearch = 0;
patchMatchParams.partialComparison = 1;
patchMatchParams.nbItersPatchMatch = 10;
patchMatchParams.patchIndexing = 0;
patchMatchParams.reconstructionType = reconstructionType;
%parameters concerning the iterations of the nearest neighbour search
%and reconstruction
maxNbIterations = 20;
residualThresh = 0.1;
%%%% get pyramid volumes
imgVolPyramid = get_image_volume_pyramid(imgVolIn,filterSize,sigma,maxLevel,size(imgVolIn,4));%patchSize(3)
occVolPyramid = get_image_volume_pyramid(occVolIn,filterSize,sigma,maxLevel,size(imgVolIn,4));%patchSize(3)
%%%%% create texture feature pyramid
if (textureFeaturesActivated>0)
disp('Calculating texture feature pyramids');
featurePyramid = get_video_features(imgVolIn,occVolIn,maxLevel,file);
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% iterate over all pyramid levels
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
for ii=maxLevel:-1:1
pp=1;
residual = inf;
occVol = single(occVolPyramid{ii});
imgVol(1,:,:,:) = imgVolPyramid{ii,1}; %get the red channel for this image volume
imgVol(2,:,:,:) = imgVolPyramid{ii,2}; %get the blue channel for this image volume
imgVol(3,:,:,:) = imgVolPyramid{ii,3}; %get the green channel for this image volume
if (exist('featurePyramid','var'))
gradX = single(featurePyramid{ii,1});
gradY = single(featurePyramid{ii,2});
normGradX = single(featurePyramid{ii,3});
normGradY = single(featurePyramid{ii,4});
patchMatchParams.gradX = gradX;
patchMatchParams.gradY = gradY;
patchMatchParams.normGradX = normGradX;
patchMatchParams.normGradY = normGradY;
end
% If we are not at the coarsest level...
if (ii~=maxLevel)
% ... upsample the previous shift map...
shiftVol = single(interpolate_disp_field(shiftVol,imgVol,1/scaleStep, patchSize,'nearest'));
% ... and recreate image volume.
if (exist('featurePyramid','var'))
[imgVol,gradX,gradY,normGradX,normGradY] = ...
reconstruct_video_and_features_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
patchMatchParams.gradX = single(gradX);
patchMatchParams.gradY = single(gradY);
patchMatchParams.normGradX = single(normGradX);
patchMatchParams.normGradY = single(normGradY);
else
[imgVol] = reconstruct_video_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
end
end
% Dilate the occlusion
structElPatch = strel('arbitrary', ones(patchSize(2),patchSize(1),patchSize(3)));
occVolDilate = imdilate(occVol,structElPatch);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% start of iterative inpainting at this level
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
while (pp <= maxNbIterations && residual > residualThresh)
imgVolIterMinusOne = imgVol+1-1;
if (ii ~= maxLevel || pp>1) %not bottom level
patchMatchParams.partialComp = 0;
patchMatchParams.partialComparison = 0;
useAllPatches = 1;
% Copy last shiftVol
firstGuess = shiftVol+1-1;
% Carry out the 3D PatchMatch
shiftVol = spatio_temporal_patch_match_mex(...
imgVol,...
imgVol,...
patchMatchParams,...
firstGuess,...
occVolDilate,...
occVolDilate);
% endfile = strcat('_',strcat(int2str(100*(maxLevel-ii+1)+pp-1),'.csv'));
%
% cd matlab_debug
%
% [~,w,l,maxFrame] = size(shiftVol);
%
% for nbframe=1:maxFrame
% csvwrite(strcat('M_DispX_',strcat(int2str(nbframe),endfile)),reshape(shiftVol(1,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_DispY_',strcat(int2str(nbframe),endfile)),reshape(shiftVol(2,:,:,nbframe),[w,l]));
% csvwrite(strcat('M_DispZ_',strcat(int2str(nbframe),endfile)),reshape(shiftVol(3,:,:,nbframe),[w,l]));
% end
%
% cd ..
% Carry out the reconstruction
if (exist('featurePyramid','var'))
[imgVol,gradX,gradY,normGradX,normGradY] = ...
reconstruct_video_and_features_mex(...
imgVol,...
occVol,...
shiftVol,patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
patchMatchParams.gradX = single(gradX);
patchMatchParams.gradY = single(gradY);
patchMatchParams.normGradX = single(normGradX);
patchMatchParams.normGradY = single(normGradY);
else
[imgVol] = reconstruct_video_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
end
else
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% INITIALISATION %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
occVolIter = occVol;
debug_iter =0;
%fill in, in an onion peel fashion
while(sum(occVolIter(:)) > 0)
structElCube = strel('arbitrary', ones(3,3,3));
occVolErode = imerode(occVolIter,structElCube);
%set up the partial occlusion volume for the PatchMatch :
% - 0 for non-occlusion;
% - 1 for occluded and not to take into account when
% comparing patches
% - 2 for occluded and to take into account (we do not allow
%the nearest neighbours to point to these pixels, but
%they have been reconstructed at this iteration)
occVolPatchMatch = occVolDilate;
occVolPatchMatch((occVolDilate - occVolIter) == 1) = 2;
%initial guess : by default, set everything to 0
if (~exist('shiftVol','var'))
firstGuess = single(zeros([4 size(imgVol,2) size(imgVol,3) size(imgVol,4)]));
else
firstGuess = shiftVol+1-1;
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%carry out the 3D patchMatch
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
shiftVol = ...
spatio_temporal_patch_match_mex(...
imgVol,...
imgVol,...
patchMatchParams,...
firstGuess,...
occVolPatchMatch,...
occVolDilate);
%determine the pixels to reconstruct at this layer
occVolBorder = abs(occVolIter - occVolErode);
%if the occVol == 2, then we cannot use the colour
%info, but we do not inpaint it either
occVolBorder(occVolErode == 1) = 2;
cd matlab_debug
[~,w,l,maxFrame] = size(shiftVol);
write_image_volume(O
for nbframe=1:maxFrame
csvwrite(strcat('M_Init_DispX_',strcat(int2str(nbframe),'.csv')),reshape(shiftVol(1,:,:,nbframe),[w,l]));
csvwrite(strcat('M_Init_DispY_',strcat(int2str(nbframe),'.csv')),reshape(shiftVol(2,:,:,nbframe),[w,l]));
csvwrite(strcat('M_Init_DispZ_',strcat(int2str(nbframe),'.csv')),reshape(shiftVol(3,:,:,nbframe),[w,l]));
end
cd ..
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%carry out the reconstruction
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if (exist('featurePyramid','var') && exist('shiftVol','var'))
[imgVol,gradX,gradY,normGradX,normGradY] = ...
reconstruct_video_and_features_mex(...
imgVol,...
occVolBorder,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
patchMatchParams.gradX = single(gradX);
patchMatchParams.gradY = single(gradY);
patchMatchParams.normGradX = single(normGradX);
patchMatchParams.normGradY = single(normGradY);
else
[imgVol] = reconstruct_video_mex(...
imgVol,...
occVolBorder,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,...
reconstructionType);
end
%now go to the next onion peel layer
occVolIter = occVolErode;
debug_iter = debug_iter +1;
end
end
pp=pp+1;
%we have finished the initialisation : make sure that both
%the patchMatch AND the reconstruction consider that
%everything is known (no more partial patch comparisons)
patchMatchParams.partialComparison = 0;
patchMatchParams.useAllPatches = 1;
useAllPatches = 1;
%calculate the residual to see if we terminate
residual = sum(abs(imgVolIterMinusOne(:) - imgVol(:)))/(single(3*sum(occVol(:)>0)));
end
if (ii==1)
imgVol = reconstruct_video_mex(...
imgVol,...
occVol,...
shiftVol,...
patchMatchParams,...
sigmaColour,...
useAllPatches,1);
% occInds = find(occVol>0);
% energy = sum(shiftVol(occInds + 3*prod(sizeImgVol(1:2))));
imgOut = imgVol;
shiftVolOut = shiftVol;
size(shiftVolOut)
% cd ../Data;
% write_image_volume(shiftVolOut,strcat('disp',str((maxLevel-ii)*100 + pp);
% cd ../MATLAB Inpainting
return;
end
%erase some of the video structures
clearvars imgVol occVolDilate occVolPatchMatch;
patchMatchParams.partialComparison = 0;
useAllPatches = 1;
end
imgOut = imgVol;
end
compare.py
View file @ 7a9222f9
......@@ -19,6 +19,7 @@ Cpp30 = "Data/onion_debug/CppOut30F"
PeelMatlab = "Data/onion_debug/peel_compare/M_EndPeel_F_frame_"
PeelCpp = "Data/onion_debug/peel_compare/Cpp_EndPeel_F"
OccSmallPath = "Data/onion_debug/Occ/OccF0.png"
CppLoadingPath = "Data/onion_debug/CppOut_WithMatlabInput_F"
# print(OccMask)
threshold_nbPixDiff = 0.1
......@@ -178,6 +179,7 @@ if __name__ == '__main__':
VolCpp30 = np.zeros((98,68,264,3))
VolPeelM = np.zeros((98,17,66,3))
VolPeelC = np.zeros((98,17,66,3))
VolCppLoad = np.zeros((98,68,264,3))
# Chargement des images dans un volume
for i in range(98):
if(i<9):
......@@ -194,6 +196,7 @@ if __name__ == '__main__':
# img_cpp30 = io.imread(Cpp30+str(i)+".png")
img_peelM = io.imread(PeelMatlab+str(i)+".png")
img_peelC = io.imread(PeelCpp+str(i)+".png")
img_load = io.imread(CppLoadingPath+str(i)+".png")
# print(img_cpp.shape)
......@@ -206,7 +209,7 @@ if __name__ == '__main__':
# VolCpp30[i,:,:,:]=img_cpp30
VolPeelC[i,:,:,:]=img_peelC
VolPeelM[i,:,:,:]=img_peelM
VolCppLoad[i,:,:,:]=img_load
# # Code for video
# cap = cv2.VideoCapture(OriFilePath)
# i=0
......@@ -241,17 +244,33 @@ if __name__ == '__main__':
# # Affichage résultat de la comparaison des pixels dans la zone du masque
# pixCountRGB = np.zeros((3,98))
# for i in range(98):
# pixCountRGB[:,i]=PixCompare(VolPeelM[i,:,:,:],VolPeelC[i,:,:,:],OccSmallMask,0.1)
# pixCountRGB[:,i]=PixCompare(VolCpp[i,:,:,:],VolCppLoad[i,:,:,:],OccMask,0.1)
#
# plt.plot(pixCountRGB[0,:],'r', label='Red pixel ratio')
# plt.plot(pixCountRGB[1,:],'g',label='Green pixel ratio')
# plt.plot(pixCountRGB[2,:],'b',label='Blue pixel ratio')
# plt.plot((pixCountRGB[0,:]+pixCountRGB[1,:]+pixCountRGB[2,:])/3,'black',label='Mean pixel ratio')
# plt.axhline(0.05)
# # plt.axhline(0.05)
# plt.xlabel("number of frame (time)")
# plt.ylabel("Percentage of pixels different in each image on the mask")
# plt.legend()
# plt.title("Evolution de la différence de pixel entre le code C++ et Matlab à la fin de l'initialisation en pelure d'onion")
# plt.title("Evolution de la différence de pixel entre le code C++ avec et sans initialisation en pelure d'onion")
# plt.show()
# # Affichage résultat de la comparaison des pixels dans la zone du masque
# pixCountCpp = np.zeros((3,98))
# pixCountLoad = np.zeros((3,98))
# for i in range(98):
# pixCountCpp[:,i]=PixCompare(VolCpp[i,:,:,:],VolMatlab[i,:,:,:],OccMask,0.1)
# pixCountLoad[:,i]=PixCompare(VolCppLoad[i,:,:,:],VolMatlab[i,:,:,:],OccMask,0.1)
#
# plt.plot((pixCountCpp[0,:]+pixCountCpp[1,:]+pixCountCpp[2,:])/3,'black',label='Cpp with peel init')
# plt.plot((pixCountLoad[0,:]+pixCountLoad[1,:]+pixCountLoad[2,:])/3,'blue',label='Cpp with matlab data loading')
# # plt.axhline(0.05)
# plt.xlabel("number of frame (time)")
# plt.ylabel("Percentage of pixels different in each image on the mask")
# plt.legend()
# plt.title("Evolution de la différence de pixel entre le code C++ avec et sans initialisation en pelure d'onion")
# plt.show()
# ## Evolution du PSNR dans la vidéo
......@@ -283,6 +302,19 @@ if __name__ == '__main__':
# plt.title("Comparaison of mean PSNR over RGB during time for differents search iterations in the C++ algorithm")
# plt.show()
## PSNR entre Cpp avec init_peel ou en chargeant les données matlab (reference avec Matlab)
# PSNR_vol = np.zeros((2,98))
# for i in range(98):
# [_,_,_,PSNR_vol[0,i]] = PSNR(VolMatlab[i,:,:,:],VolCpp[i,:,:,:],OccMask)
# [_,_,_,PSNR_vol[1,i]] = PSNR(VolMatlab[i,:,:,:],VolCppLoad[i,:,:,:],OccMask)
# plt.plot(PSNR_vol[0,:],'r', label='Cpp with onion peel init')
# plt.plot(PSNR_vol[1,:],'g',label='Cpp with loading matlab data')
# plt.ylabel("PSNR on the mask during time")
# plt.xlabel("number of frame (time)")
# plt.legend()
# plt.title("Comparaison of mean PSNR over RGB during time for C++ algorithm with or without self onion peel initialisation")
# plt.show()
# ## Evolution du SSIM dans la vidéo en comparaison CPP 10/20/30 search iter
# SSIM_cpp10 = np.zeros((3,98))
# SSIM_cpp =np.zeros((3,98))
......@@ -325,4 +357,4 @@ if __name__ == '__main__':
# plt.show()
# VolOutComp = visual_compare(VolRand1,VolRand2,"Data/random_compare/RComp")
visual_compare(VolPeelM,VolPeelC,"Data/onion_debug/peel_compare/CompF")
visual_compare(VolCpp,VolCppLoad,"Data/onion_debug/peel_compare/visual_comp_cpp_matlab_loading/CompLoadF")
onion.py
View file @ 7a9222f9
......@@ -84,6 +84,6 @@ p1 = Image.fromarray(np.uint8(p_temp))
for i in range(98):
p1 = Image.open("Data/Out_matlab_random/beach_umbrellainpainted_frame_0"+str(i)+".png")
p2 = Image.open("Data/onion_debug/CppOutF"+str(i)+".png")
p2 = Image.open("Data/onion_debug/CppOut_WithMatlabInput_F"+str(i)+".png")
p3 = Image.blend(p1,p2,.8)
p3.save("Data/onion_debug/WTF_"+str(i)+".png")
p3.save("Data/onion_debug/Merge_"+str(i)+".png")
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment