External denoiser based on noise estimation and moving object detection.

webrtc/modules/video_processing/video_denoiser.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/common_video/libyuv/include/scaler.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/modules/video_processing/video_denoiser.h"
 
 namespace webrtc {
 
 VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection)
     : width_(0),
       height_(0),
-      filter_(DenoiserFilter::Create(runtime_cpu_detection)) {}
+      filter_(DenoiserFilter::Create(runtime_cpu_detection)),
+      ne_(new NoiseEstimation()) {}
 
-void VideoDenoiser::TrailingReduction(int mb_rows,
-                                      int mb_cols,
-                                      const uint8_t* y_src,
-                                      int stride_y,
-                                      uint8_t* y_dst) {
-  for (int mb_row = 1; mb_row < mb_rows - 1; ++mb_row) {
-    for (int mb_col = 1; mb_col < mb_cols - 1; ++mb_col) {
-      int mb_index = mb_row * mb_cols + mb_col;
-      uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
-      const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
-      // If the number of denoised neighbors is less than a threshold,
-      // do NOT denoise for the block. Set different threshold for skin MB.
-      // The change of denoising status will not propagate.
-      if (metrics_[mb_index].is_skin) {
-        // The threshold is high (more strict) for non-skin MB where the
-        // trailing usually happen.
-        if (metrics_[mb_index].denoise &&
-            metrics_[mb_index + 1].denoise + metrics_[mb_index - 1].denoise +
-                    metrics_[mb_index + mb_cols].denoise +
-                    metrics_[mb_index - mb_cols].denoise <=
-                2) {
-          metrics_[mb_index].denoise = 0;
-          filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
-        }
-      } else if (metrics_[mb_index].denoise &&
-                 metrics_[mb_index + 1].denoise +
-                         metrics_[mb_index - 1].denoise +
-                         metrics_[mb_index + mb_cols + 1].denoise +
-                         metrics_[mb_index + mb_cols - 1].denoise +
-                         metrics_[mb_index - mb_cols + 1].denoise +
-                         metrics_[mb_index - mb_cols - 1].denoise +
-                         metrics_[mb_index + mb_cols].denoise +
-                         metrics_[mb_index - mb_cols].denoise <=
-                     7) {
-        filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
-      }
-    }
+#if EXPERIMENTAL
+// Draft. This should be replaced with lib function.
+static void CopyStatus(std::unique_ptr<uint8_t[]>* d1,
+                       const std::unique_ptr<uint8_t[]>& d2,
+                       int len) {
+  for (int i = 0; i < len; ++i) {
+    (*d1)[i] = d2[i];
   }
 }
 
+// Check the mb position(1: close to the center, 3: close to the border).
+static int PositionCheck(int mb_row, int mb_col, int mb_rows, int mb_cols) {
+  if ((mb_row >= (mb_rows >> 3)) && (mb_row <= (7 * mb_rows >> 3)) &&
+      (mb_col >= (mb_cols >> 3)) && (mb_col <= (7 * mb_cols >> 3)))
+    return 1;
+  else if ((mb_row >= (mb_rows >> 4)) && (mb_row <= (15 * mb_rows >> 4)) &&
+           (mb_col >= (mb_cols >> 4)) && (mb_col <= (15 * mb_cols >> 4)))
+    return 2;
+  else
+    return 3;
+}
+#endif
+
 void VideoDenoiser::DenoiseFrame(const VideoFrame& frame,
-                                 VideoFrame* denoised_frame) {
+                                 VideoFrame* denoised_frame,
+                                 VideoFrame* denoised_frame_track) {
   int stride_y = frame.stride(kYPlane);
   int stride_u = frame.stride(kUPlane);
   int stride_v = frame.stride(kVPlane);
   // If previous width and height are different from current frame's, then no
   // denoising for the current frame.
   if (width_ != frame.width() || height_ != frame.height()) {
     width_ = frame.width();
     height_ = frame.height();
     denoised_frame->CreateFrame(frame.buffer(kYPlane), frame.buffer(kUPlane),
                                 frame.buffer(kVPlane), width_, height_,
                                 stride_y, stride_u, stride_v, kVideoRotation_0);
+    denoised_frame_track->CreateFrame(
+        frame.buffer(kYPlane), frame.buffer(kUPlane), frame.buffer(kVPlane),
+        width_, height_, stride_y, stride_u, stride_v, kVideoRotation_0);
     // Setting time parameters to the output frame.
     denoised_frame->set_timestamp(frame.timestamp());
     denoised_frame->set_render_time_ms(frame.render_time_ms());
+    ne_->SetResolution(width_, height_);
     return;
   }
   // For 16x16 block.
   int mb_cols = width_ >> 4;
   int mb_rows = height_ >> 4;
   if (metrics_.get() == nullptr)
     metrics_.reset(new DenoiseMetrics[mb_cols * mb_rows]());
+  if (d_status_.get() == nullptr) {
+    d_status_.reset(new uint8_t[mb_cols * mb_rows]());
+#if EXPERIMENTAL
+    d_status_tmp1_.reset(new uint8_t[mb_cols * mb_rows]());
+    d_status_tmp2_.reset(new uint8_t[mb_cols * mb_rows]());
+#endif
+    x_density_.reset(new uint8_t[mb_cols]());
+    y_density_.reset(new uint8_t[mb_rows]());
+  }
+
   // Denoise on Y plane.
   uint8_t* y_dst = denoised_frame->buffer(kYPlane);
   uint8_t* u_dst = denoised_frame->buffer(kUPlane);
   uint8_t* v_dst = denoised_frame->buffer(kVPlane);
+  uint8_t* y_dst_track = denoised_frame_track->buffer(kYPlane);
   const uint8_t* y_src = frame.buffer(kYPlane);
   const uint8_t* u_src = frame.buffer(kUPlane);
   const uint8_t* v_src = frame.buffer(kVPlane);
+  uint8_t noise_level = ne_->GetNoiseLevel();
   // Temporary buffer to store denoising result.
   uint8_t y_tmp[16 * 16] = {0};
   for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
+    y_density_[mb_row] = 0;
+  }
+  for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {

[marpan, 2016/03/24 20:56:16]

can you use memset here?

[jackychen_, 2016/03/25 18:45:44]

Done.

+    x_density_[mb_col] = 0;
+  }
+  for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {

[marpan, 2016/03/24 20:56:16]

put more comments to explain what this block of code does.
i.e, something like: "...Loop over blocks, compute variance between ?? and ??,
accumulate/extract noise level, and update x/y_density factors for moving object
extraction...."

[jackychen_, 2016/03/25 18:45:44]

Done.

+    for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
+      const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
+      uint8_t* mb_dst_track =
+          y_dst_track + (mb_row << 4) * stride_y + (mb_col << 4);
+      int mb_index = mb_row * mb_cols + mb_col;
+#if EXPERIMENTAL
+      int pos_factor = PositionCheck(mb_row, mb_col, mb_rows, mb_cols);
+      uint32_t thr_var_adp = 16 * 16 * 5 * (noise_level ? pos_factor : 1);
+#else
+      uint32_t thr_var_adp = 16 * 16 * 5 * (noise_level + 1);
+#endif
+      int brightness = 0;
+      for (int i = 0; i < 16; ++i) {
+        for (int j = 0; j < 16; ++j) {
+          brightness += mb_src[i * stride_y + j];
+        }
+      }
+
+      // Get the averaged block.

[marpan, 2016/03/30 18:27:19]

Get denoised block

+      filter_->MbDenoise(mb_dst_track, stride_y, y_tmp, 16, mb_src, stride_y, 0,
+                         1, true);
+      metrics_[mb_index].var = filter_->Variance16x8(
+          mb_dst_track, stride_y, y_tmp, 16, &metrics_[mb_index].sad);
+
+      if (metrics_[mb_index].var > thr_var_adp) {
+        ne_->ResetConsecLowVar(mb_index);
+        d_status_[mb_index] = 1;
+#if EXPERIMENTAL
+        if (noise_level == 0 || pos_factor < 3) {
+          x_density_[mb_col] += 1;
+          y_density_[mb_row] += 1;
+        }
+#else
+        x_density_[mb_col] += 1;
+        y_density_[mb_row] += 1;
+#endif
+      } else {
+        uint32_t sse_t = 0;
+        uint32_t noise_var = filter_->Variance16x8(mb_dst_track, stride_y,
+                                                   mb_src, stride_y, &sse_t);
+        ne_->GetNoise(mb_index, noise_var, brightness);
+        d_status_[mb_index] = 0;
+      }
+      // Track averaged frame.

[marpan, 2016/03/24 20:56:16]

Track denoised frame?

[jackychen_, 2016/03/25 18:45:45]

Done.

+      filter_->CopyMem16x16(y_tmp, 16, mb_dst_track, stride_y);
+    }
+  }
+
+#if EXPERIMENTAL
+  // Draft. This can be optimized.
+  int mb_row_min = noise_level ? mb_rows >> 3 : 1;
+  int mb_col_min = noise_level ? mb_cols >> 3 : 1;
+  int mb_row_max = noise_level ? (7 * mb_rows >> 3) : mb_rows - 2;
+  int mb_col_max = noise_level ? (7 * mb_cols >> 3) : mb_cols - 2;
+  CopyStatus(&d_status_tmp1_, d_status_, mb_rows * mb_cols);
+  // Up left.
+  for (int mb_row = mb_row_min; mb_row <= mb_row_max; ++mb_row) {
+    for (int mb_col = mb_col_min; mb_col <= mb_col_max; ++mb_col) {
+      d_status_tmp1_[mb_row * mb_cols + mb_col] |=
+          (d_status_tmp1_[(mb_row - 1) * mb_cols + mb_col] |
+           d_status_tmp1_[mb_row * mb_cols + mb_col - 1]);
+    }
+  }
+  CopyStatus(&d_status_tmp2_, d_status_tmp1_, mb_rows * mb_cols);
+  CopyStatus(&d_status_tmp1_, d_status_, mb_rows * mb_cols);
+  // Bottom left.
+  for (int mb_row = mb_row_max; mb_row >= mb_row_min; --mb_row) {
+    for (int mb_col = mb_col_min; mb_col <= mb_col_max; ++mb_col) {
+      d_status_tmp1_[mb_row * mb_cols + mb_col] |=
+          (d_status_tmp1_[(mb_row + 1) * mb_cols + mb_col] |
+           d_status_tmp1_[mb_row * mb_cols + mb_col - 1]);
+      d_status_tmp2_[mb_row * mb_cols + mb_col] &=
+          d_status_tmp1_[mb_row * mb_cols + mb_col];
+    }
+  }
+  CopyStatus(&d_status_tmp1_, d_status_, mb_rows * mb_cols);
+  // Up right.
+  for (int mb_row = mb_row_min; mb_row <= mb_row_max; ++mb_row) {
+    for (int mb_col = mb_col_max; mb_col >= mb_col_min; --mb_col) {
+      d_status_tmp1_[mb_row * mb_cols + mb_col] |=
+          (d_status_tmp1_[(mb_row - 1) * mb_cols + mb_col] |
+           d_status_tmp1_[mb_row * mb_cols + mb_col + 1]);
+      d_status_tmp2_[mb_row * mb_cols + mb_col] &=
+          d_status_tmp1_[mb_row * mb_cols + mb_col];
+    }
+  }
+  CopyStatus(&d_status_tmp1_, d_status_, mb_rows * mb_cols);
+  // Bottom right.
+  for (int mb_row = mb_row_max; mb_row >= mb_row_min; --mb_row) {
+    for (int mb_col = mb_col_max; mb_col >= mb_col_min; --mb_col) {
+      d_status_tmp1_[mb_row * mb_cols + mb_col] |=
+          (d_status_tmp1_[(mb_row + 1) * mb_cols + mb_col] |
+           d_status_tmp1_[mb_row * mb_cols + mb_col + 1]);
+      d_status_tmp2_[mb_row * mb_cols + mb_col] &=
+          d_status_tmp1_[mb_row * mb_cols + mb_col];
+    }
+  }
+#endif
+
+  for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {

[marpan, 2016/03/24 20:56:16]

put comments to explain this block of code

[jackychen_, 2016/03/25 18:45:45]

Done.

     for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
       const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
       uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
-      int mb_index = mb_row * mb_cols + mb_col;
-      // Denoise each MB at the very start and save the result to a temporary
-      // buffer.
-      if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0,
-                             1) == FILTER_BLOCK) {
-        uint32_t thr_var = 0;
-        // Save var and sad to the buffer.
-        metrics_[mb_index].var = filter_->Variance16x8(
-            mb_dst, stride_y, y_tmp, 16, &metrics_[mb_index].sad);
-        // Get skin map.
-        metrics_[mb_index].is_skin = MbHasSkinColor(
-            y_src, u_src, v_src, stride_y, stride_u, stride_v, mb_row, mb_col);
-        // Variance threshold for skin/non-skin MB is different.
-        // Skin MB use a small threshold to reduce blockiness.
-        thr_var = metrics_[mb_index].is_skin ? 128 : 12 * 128;
-        if (metrics_[mb_index].var > thr_var) {
-          metrics_[mb_index].denoise = 0;
-          // Use the source MB.
-          filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
-        } else {
-          metrics_[mb_index].denoise = 1;
-          // Use the denoised MB.
-          filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y);
-        }
-      } else {
-        metrics_[mb_index].denoise = 0;
-        filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
-      }
-      // Copy source U/V plane.
       const uint8_t* mb_src_u =
           u_src + (mb_row << 3) * stride_u + (mb_col << 3);
       const uint8_t* mb_src_v =
           v_src + (mb_row << 3) * stride_v + (mb_col << 3);
       uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3);
       uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3);
+#if EXPERIMENTAL
+      if (!d_status_tmp2_[mb_row * mb_cols + mb_col] ||
+          x_density_[mb_col] * y_density_[mb_row] == 0) {
+#else
+      if (x_density_[mb_col] * y_density_[mb_row] == 0) {
+#endif
+        if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0,
+                               noise_level, false) == FILTER_BLOCK) {
+          filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y);
+        } else {
+          // Copy y source.
+          filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
+        }
+      } else {
+        // Copy y source.
+        filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
+      }
       filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
       filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
     }
   }
-  // Second round.
-  // This is to reduce the trailing artifact and blockiness by referring
-  // neighbors' denoising status.
-  TrailingReduction(mb_rows, mb_cols, y_src, stride_y, y_dst);
+
+#if DISPLAY  // Rectangle diagnostics
+  // Show rectangular region
+  for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
+    for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
+      int mb_index = mb_row * mb_cols + mb_col;
+      const uint8_t* mb_src_u =
+          u_src + (mb_row << 3) * stride_u + (mb_col << 3);
+      const uint8_t* mb_src_v =
+          v_src + (mb_row << 3) * stride_v + (mb_col << 3);
+      uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3);
+      uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3);
+      uint8_t y_tmp_255[8 * 8];
+      memset(y_tmp_255, 200, 8 * 8);
+      // x_density_[mb_col] * y_density_[mb_row]
+      if (d_status_[mb_index] == 1) {
+        // Paint to red.
+        filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
+        filter_->CopyMem8x8(y_tmp_255, 8, mb_dst_v, stride_v);
+      } else if (d_status_tmp2_[mb_row * mb_cols + mb_col] &&
+                 x_density_[mb_col] * y_density_[mb_row]) {
+        // Paint to blue.
+        filter_->CopyMem8x8(y_tmp_255, 8, mb_dst_u, stride_u);
+        filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
+      } else {
+        filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
+        filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
+      }
+    }
+  }
+#endif
 
   // Setting time parameters to the output frame.
   denoised_frame->set_timestamp(frame.timestamp());
   denoised_frame->set_render_time_ms(frame.render_time_ms());
   return;
 }
 
 }  // namespace webrtc