Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(496)

Unified Diff: webrtc/modules/video_processing/video_denoiser.cc

Issue 1822333003: External denoiser based on noise estimation and moving object detection. (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@master
Patch Set: Created 4 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: webrtc/modules/video_processing/video_denoiser.cc
diff --git a/webrtc/modules/video_processing/video_denoiser.cc b/webrtc/modules/video_processing/video_denoiser.cc
index 3951381d22c00ae40d06b5730f90cae0fdd38763..b00da5c90a16c9808149d3c1b562165492a752c2 100644
--- a/webrtc/modules/video_processing/video_denoiser.cc
+++ b/webrtc/modules/video_processing/video_denoiser.cc
@@ -16,50 +16,144 @@ namespace webrtc {
VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection)
: width_(0),
height_(0),
- filter_(DenoiserFilter::Create(runtime_cpu_detection)) {}
-
-void VideoDenoiser::TrailingReduction(int mb_rows,
- int mb_cols,
- const uint8_t* y_src,
- int stride_y,
- uint8_t* y_dst) {
- for (int mb_row = 1; mb_row < mb_rows - 1; ++mb_row) {
- for (int mb_col = 1; mb_col < mb_cols - 1; ++mb_col) {
+ filter_(DenoiserFilter::Create(runtime_cpu_detection, &cpu_type_)),
+ ne_(new NoiseEstimation()) {}
+
+#if EXPERIMENTAL
+// Check the mb position(1: close to the center, 3: close to the border).
+static int PositionCheck(int mb_row, int mb_col, int mb_rows, int mb_cols) {
+ if ((mb_row >= (mb_rows >> 3)) && (mb_row <= (7 * mb_rows >> 3)) &&
+ (mb_col >= (mb_cols >> 3)) && (mb_col <= (7 * mb_cols >> 3)))
+ return 1;
+ else if ((mb_row >= (mb_rows >> 4)) && (mb_row <= (15 * mb_rows >> 4)) &&
+ (mb_col >= (mb_cols >> 4)) && (mb_col <= (15 * mb_cols >> 4)))
+ return 2;
+ else
+ return 3;
+}
+
+static void ReduceFalseDetection(const std::unique_ptr<uint8_t[]>& d_status,
+ std::unique_ptr<uint8_t[]>* d_status_tmp1,
+ std::unique_ptr<uint8_t[]>* d_status_tmp2,
+ int noise_level,
+ int mb_rows,
+ int mb_cols) {
+ // Draft. This can be optimized. This code block is to reduce false detection
+ // in moving object detection.
+ int mb_row_min = noise_level ? mb_rows >> 3 : 1;
+ int mb_col_min = noise_level ? mb_cols >> 3 : 1;
+ int mb_row_max = noise_level ? (7 * mb_rows >> 3) : mb_rows - 2;
+ int mb_col_max = noise_level ? (7 * mb_cols >> 3) : mb_cols - 2;
+ memcpy((*d_status_tmp1).get(), d_status.get(), mb_rows * mb_cols);
+ // Up left.
+ for (int mb_row = mb_row_min; mb_row <= mb_row_max; ++mb_row) {
+ for (int mb_col = mb_col_min; mb_col <= mb_col_max; ++mb_col) {
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col] |=
+ ((*d_status_tmp1)[(mb_row - 1) * mb_cols + mb_col] |
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col - 1]);
+ }
+ }
+ memcpy((*d_status_tmp2).get(), (*d_status_tmp1).get(), mb_rows * mb_cols);
+ memcpy((*d_status_tmp1).get(), d_status.get(), mb_rows * mb_cols);
+ // Bottom left.
+ for (int mb_row = mb_row_max; mb_row >= mb_row_min; --mb_row) {
+ for (int mb_col = mb_col_min; mb_col <= mb_col_max; ++mb_col) {
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col] |=
+ ((*d_status_tmp1)[(mb_row + 1) * mb_cols + mb_col] |
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col - 1]);
+ (*d_status_tmp2)[mb_row * mb_cols + mb_col] &=
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col];
+ }
+ }
+ memcpy((*d_status_tmp1).get(), d_status.get(), mb_rows * mb_cols);
+ // Up right.
+ for (int mb_row = mb_row_min; mb_row <= mb_row_max; ++mb_row) {
+ for (int mb_col = mb_col_max; mb_col >= mb_col_min; --mb_col) {
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col] |=
+ ((*d_status_tmp1)[(mb_row - 1) * mb_cols + mb_col] |
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col + 1]);
+ (*d_status_tmp2)[mb_row * mb_cols + mb_col] &=
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col];
+ }
+ }
+ memcpy((*d_status_tmp1).get(), d_status.get(), mb_rows * mb_cols);
+ // Bottom right.
+ for (int mb_row = mb_row_max; mb_row >= mb_row_min; --mb_row) {
+ for (int mb_col = mb_col_max; mb_col >= mb_col_min; --mb_col) {
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col] |=
+ ((*d_status_tmp1)[(mb_row + 1) * mb_cols + mb_col] |
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col + 1]);
+ (*d_status_tmp2)[mb_row * mb_cols + mb_col] &=
+ (*d_status_tmp1)[mb_row * mb_cols + mb_col];
+ }
+ }
+}
+
+static bool TrailingBlock(const std::unique_ptr<uint8_t[]>& d_status,
+ int mb_row,
+ int mb_col,
+ int mb_rows,
+ int mb_cols) {
+ int mb_index = mb_row * mb_cols + mb_col;
+ if (!mb_row || !mb_col || mb_row == mb_rows - 1 || mb_col == mb_cols - 1)
+ return false;
+ return d_status[mb_index + 1] || d_status[mb_index - 1] ||
+ d_status[mb_index + mb_cols] || d_status[mb_index - mb_cols];
+}
+#endif
+
+#if DISPLAY
+void ShowRect(const std::unique_ptr<DenoiserFilter>& filter,
+ const std::unique_ptr<uint8_t[]>& d_status,
+ const std::unique_ptr<uint8_t[]>& d_status_tmp2,
+ const std::unique_ptr<uint8_t[]>& x_density,
+ const std::unique_ptr<uint8_t[]>& y_density,
+ const uint8_t* u_src,
+ const uint8_t* v_src,
+ uint8_t* u_dst,
+ uint8_t* v_dst,
+ int mb_rows,
+ int mb_cols,
+ int stride_u,
+ int stride_v) {
+ for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
+ for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
int mb_index = mb_row * mb_cols + mb_col;
- uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
- const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
- // If the number of denoised neighbors is less than a threshold,
- // do NOT denoise for the block. Set different threshold for skin MB.
- // The change of denoising status will not propagate.
- if (metrics_[mb_index].is_skin) {
- // The threshold is high (more strict) for non-skin MB where the
- // trailing usually happen.
- if (metrics_[mb_index].denoise &&
- metrics_[mb_index + 1].denoise + metrics_[mb_index - 1].denoise +
- metrics_[mb_index + mb_cols].denoise +
- metrics_[mb_index - mb_cols].denoise <=
- 2) {
- metrics_[mb_index].denoise = 0;
- filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
- }
- } else if (metrics_[mb_index].denoise &&
- metrics_[mb_index + 1].denoise +
- metrics_[mb_index - 1].denoise +
- metrics_[mb_index + mb_cols + 1].denoise +
- metrics_[mb_index + mb_cols - 1].denoise +
- metrics_[mb_index - mb_cols + 1].denoise +
- metrics_[mb_index - mb_cols - 1].denoise +
- metrics_[mb_index + mb_cols].denoise +
- metrics_[mb_index - mb_cols].denoise <=
- 7) {
- filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
+ const uint8_t* mb_src_u =
+ u_src + (mb_row << 3) * stride_u + (mb_col << 3);
+ const uint8_t* mb_src_v =
+ v_src + (mb_row << 3) * stride_v + (mb_col << 3);
+ uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3);
+ uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3);
+ uint8_t y_tmp_255[8 * 8];
+ memset(y_tmp_255, 200, 8 * 8);
+ // x_density_[mb_col] * y_density_[mb_row]
+ if (d_status[mb_index] == 1) {
+ // Paint to red.
+ filter->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
+ filter->CopyMem8x8(y_tmp_255, 8, mb_dst_v, stride_v);
+#if EXPERIMENTAL
+ } else if (d_status_tmp2[mb_row * mb_cols + mb_col] &&
+ x_density[mb_col] * y_density[mb_row]) {
+#else
+ } else if (x_density[mb_col] * y_density[mb_row]) {
+#endif
+ // Paint to blue.
+ filter->CopyMem8x8(y_tmp_255, 8, mb_dst_u, stride_u);
+ filter->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
+ } else {
+ filter->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
+ filter->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
}
}
}
}
+#endif
void VideoDenoiser::DenoiseFrame(const VideoFrame& frame,
- VideoFrame* denoised_frame) {
+ VideoFrame* denoised_frame,
+ VideoFrame* denoised_frame_prev,
+ int noise_level_prev) {
int stride_y = frame.stride(kYPlane);
int stride_u = frame.stride(kUPlane);
int stride_v = frame.stride(kVPlane);
@@ -71,9 +165,13 @@ void VideoDenoiser::DenoiseFrame(const VideoFrame& frame,
denoised_frame->CreateFrame(frame.buffer(kYPlane), frame.buffer(kUPlane),
frame.buffer(kVPlane), width_, height_,
stride_y, stride_u, stride_v, kVideoRotation_0);
+ denoised_frame_prev->CreateFrame(
+ frame.buffer(kYPlane), frame.buffer(kUPlane), frame.buffer(kVPlane),
+ width_, height_, stride_y, stride_u, stride_v, kVideoRotation_0);
// Setting time parameters to the output frame.
denoised_frame->set_timestamp(frame.timestamp());
denoised_frame->set_render_time_ms(frame.render_time_ms());
+ ne_->Init(width_, height_, cpu_type_);
return;
}
// For 16x16 block.
@@ -81,62 +179,128 @@ void VideoDenoiser::DenoiseFrame(const VideoFrame& frame,
int mb_rows = height_ >> 4;
if (metrics_.get() == nullptr)
metrics_.reset(new DenoiseMetrics[mb_cols * mb_rows]());
+ if (d_status_.get() == nullptr) {
+ d_status_.reset(new uint8_t[mb_cols * mb_rows]());
+#if EXPERIMENTAL
+ d_status_tmp1_.reset(new uint8_t[mb_cols * mb_rows]());
+ d_status_tmp2_.reset(new uint8_t[mb_cols * mb_rows]());
+#endif
+ x_density_.reset(new uint8_t[mb_cols]());
+ y_density_.reset(new uint8_t[mb_rows]());
+ }
+
// Denoise on Y plane.
uint8_t* y_dst = denoised_frame->buffer(kYPlane);
uint8_t* u_dst = denoised_frame->buffer(kUPlane);
uint8_t* v_dst = denoised_frame->buffer(kVPlane);
+ uint8_t* y_dst_prev = denoised_frame_prev->buffer(kYPlane);
const uint8_t* y_src = frame.buffer(kYPlane);
const uint8_t* u_src = frame.buffer(kUPlane);
const uint8_t* v_src = frame.buffer(kVPlane);
+ uint8_t noise_level = noise_level_prev == -1 ? 0 : ne_->GetNoiseLevel();
// Temporary buffer to store denoising result.
uint8_t y_tmp[16 * 16] = {0};
+ memset(x_density_.get(), 0, mb_cols);
+ memset(y_density_.get(), 0, mb_rows);
+
+ // Loop over blocks to accumulate/extract noise level and update x/y_density
+ // factors for moving object detection.
for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
- uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
+ uint8_t* mb_dst_prev =
+ y_dst_prev + (mb_row << 4) * stride_y + (mb_col << 4);
int mb_index = mb_row * mb_cols + mb_col;
- // Denoise each MB at the very start and save the result to a temporary
- // buffer.
- if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0,
- 1) == FILTER_BLOCK) {
- uint32_t thr_var = 0;
- // Save var and sad to the buffer.
- metrics_[mb_index].var = filter_->Variance16x8(
- mb_dst, stride_y, y_tmp, 16, &metrics_[mb_index].sad);
- // Get skin map.
- metrics_[mb_index].is_skin = MbHasSkinColor(
- y_src, u_src, v_src, stride_y, stride_u, stride_v, mb_row, mb_col);
- // Variance threshold for skin/non-skin MB is different.
- // Skin MB use a small threshold to reduce blockiness.
- thr_var = metrics_[mb_index].is_skin ? 128 : 12 * 128;
- if (metrics_[mb_index].var > thr_var) {
- metrics_[mb_index].denoise = 0;
- // Use the source MB.
- filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
- } else {
- metrics_[mb_index].denoise = 1;
- // Use the denoised MB.
- filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y);
+#if EXPERIMENTAL
+ int pos_factor = PositionCheck(mb_row, mb_col, mb_rows, mb_cols);
+ uint32_t thr_var_adp = 16 * 16 * 5 * (noise_level ? pos_factor : 1);
+#else
+ uint32_t thr_var_adp = 16 * 16 * 5;
+#endif
+ int brightness = 0;
+ for (int i = 0; i < 16; ++i) {
+ for (int j = 0; j < 16; ++j) {
+ brightness += mb_src[i * stride_y + j];
}
+ }
+
+ // Get the denoised block.
+ filter_->MbDenoise(mb_dst_prev, stride_y, y_tmp, 16, mb_src, stride_y, 0,
+ 1, true);
+ // The variance is based on the denoised blocks in time T and T-1.
+ metrics_[mb_index].var = filter_->Variance16x8(
+ mb_dst_prev, stride_y, y_tmp, 16, &metrics_[mb_index].sad);
+
+ if (metrics_[mb_index].var > thr_var_adp) {
+ ne_->ResetConsecLowVar(mb_index);
+ d_status_[mb_index] = 1;
+#if EXPERIMENTAL
+ if (noise_level == 0 || pos_factor < 3) {
+ x_density_[mb_col] += 1;
+ y_density_[mb_row] += 1;
+ }
+#else
+ x_density_[mb_col] += 1;
+ y_density_[mb_row] += 1;
+#endif
} else {
- metrics_[mb_index].denoise = 0;
- filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
+ uint32_t sse_t = 0;
+ // The variance is based on the src blocks in time T and denoised block
+ // in time T-1.
+ uint32_t noise_var = filter_->Variance16x8(mb_dst_prev, stride_y,
+ mb_src, stride_y, &sse_t);
+ ne_->GetNoise(mb_index, noise_var, brightness);
+ d_status_[mb_index] = 0;
}
- // Copy source U/V plane.
+ // Track denoised frame.
+ filter_->CopyMem16x16(y_tmp, 16, mb_dst_prev, stride_y);
+ }
+ }
+
+#if EXPERIMENTAL
+ ReduceFalseDetection(d_status_, &d_status_tmp1_, &d_status_tmp2_, noise_level,
+ mb_rows, mb_cols);
+#endif
+
+ // Denoise each MB based on the results of moving objects detection.
+ for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
+ for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
+ const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
+ uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
const uint8_t* mb_src_u =
u_src + (mb_row << 3) * stride_u + (mb_col << 3);
const uint8_t* mb_src_v =
v_src + (mb_row << 3) * stride_v + (mb_col << 3);
uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3);
uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3);
+#if EXPERIMENTAL
+ if ((!d_status_tmp2_[mb_row * mb_cols + mb_col] ||
+ x_density_[mb_col] * y_density_[mb_row] == 0) &&
+ !TrailingBlock(d_status_, mb_row, mb_col, mb_rows, mb_cols)) {
+#else
+ if (x_density_[mb_col] * y_density_[mb_row] == 0) {
+#endif
+ if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0,
+ noise_level, false) == FILTER_BLOCK) {
+ filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y);
+ } else {
+ // Copy y source.
+ filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
+ }
+ } else {
+ // Copy y source.
+ filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
+ }
filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
}
}
- // Second round.
- // This is to reduce the trailing artifact and blockiness by referring
- // neighbors' denoising status.
- TrailingReduction(mb_rows, mb_cols, y_src, stride_y, y_dst);
+
+#if DISPLAY // Rectangle diagnostics
+ // Show rectangular region
+ ShowRect(filter_, d_status_, d_status_tmp2_, x_density_, y_density_, u_src,
+ v_src, u_dst, v_dst, mb_rows, mb_cols, stride_u, stride_v);
+#endif
// Setting time parameters to the output frame.
denoised_frame->set_timestamp(frame.timestamp());
« no previous file with comments | « webrtc/modules/video_processing/video_denoiser.h ('k') | webrtc/modules/video_processing/video_processing.gypi » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698