Implement AndroidTextureBuffer::NativeToI420.

talk/app/webrtc/java/jni/native_handle_impl.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
  *     this list of conditions and the following disclaimer in the documentation
  *     and/or other materials provided with the distribution.
  *  3. The name of the author may not be used to endorse or promote products
  *     derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <android/log.h>
 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
 
+#include "talk/app/webrtc/java/jni/jni_helpers.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+#include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/keep_ref_until_done.h"
+#include "webrtc/base/logging.h"
 #include "webrtc/base/scoped_ref_ptr.h"
 
 using webrtc::NativeHandleBuffer;
 
 namespace webrtc_jni {
 
+// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD.
+static const int kBufferAlignment = 64;
+
 NativeHandleImpl::NativeHandleImpl(JNIEnv* jni,
                                    jint j_oes_texture_id,
-                                   jfloatArray j_transform_matrix)
-    : oes_texture_id(j_oes_texture_id) {
+                                   jfloatArray j_transform_matrix,
+                                   jobject helper)
+  : oes_texture_id(j_oes_texture_id),
+    surface_texture_helper(helper) {
   RTC_CHECK_EQ(16, jni->GetArrayLength(j_transform_matrix));
   jfloat* transform_matrix_ptr =
       jni->GetFloatArrayElements(j_transform_matrix, nullptr);
+  // TODO(nisse): Hold on to java float array instead, since it is

[magjed_webrtc, 2015/12/03 12:33:58]

Yes, but then you need a ScopedGlobalRef or similar. I prefer to copy the floats
for simplicity. If we want to reduce overhead, we can reduce the 4x4 matrix to
2x3.

[nisse-webrtc, 2015/12/03 13:35:20]

If we add a reference to the java array, can we delete the C++ sampling_matrix?
Or do we need both?

[magjed_webrtc, 2015/12/03 14:30:46]

Yes, then we wouldn't need the C++ sampling_matrix, but please don't add a
reference to the java array, I'm sure it will just complicate things in the
future. It's just a few bytes, like width and height, it's simpler to copy those
variables than to use global refs/reference counting.

+  // used only for callbacks into java.
   for (int i = 0; i < 16; ++i) {
     sampling_matrix[i] = transform_matrix_ptr[i];
   }
   jni->ReleaseFloatArrayElements(j_transform_matrix, transform_matrix_ptr, 0);
 }
 
 AndroidTextureBuffer::AndroidTextureBuffer(
     int width,
     int height,
     const NativeHandleImpl& native_handle,
     const rtc::Callback0<void>& no_longer_used)
     : webrtc::NativeHandleBuffer(&native_handle_, width, height),
       native_handle_(native_handle),
       no_longer_used_cb_(no_longer_used) {}
 
 AndroidTextureBuffer::~AndroidTextureBuffer() {
   no_longer_used_cb_();
 }
 
+#define TAG "AndroidTextureBuffer"
+// Alternatively: #define AT_LOG LOG(LS_ERROR)
+#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG)
+
+#define USE_CONVERT_YUV 1
+#if USE_CONVERT_YUV
 rtc::scoped_refptr<webrtc::VideoFrameBuffer>
 AndroidTextureBuffer::NativeToI420Buffer() {
-  RTC_NOTREACHED()
-      << "AndroidTextureBuffer::NativeToI420Buffer not implemented.";
-  return nullptr;
+  ALOGE << "NativeToI420Buffer called";
+  int y_width = (width()+3) / 4;
+  int uv_width = (width()+7) / 8;
+  int stride = 8 * uv_width;
+  int uv_height = (height()+1)/2;
+  size_t size = stride * (height() + uv_height);
+  // TODO(nisse): Use scoped_ptr? The data is owned by the frame, and
+  // deleted by its destructor callback.

[magjed_webrtc, 2015/12/03 12:33:58]

Yes, use scoped_ptr here...

[nisse-webrtc, 2015/12/04 09:40:34]

Done. Not sure if it's an improvement, though. 

We allocate the buffer, then pass ownership of it to the frame. And at the same
time we also wrap it in a java ByteBuffer, which hopefully isn't used beyond the
lifetime of the pixel data. Maybe it would be cleaner to arrange so that the
data is owned by the ByteBuffer, and let the frame "own" a reference to that
buffer.

+  uint8_t *y_data = static_cast<uint8_t*>(
+      webrtc::AlignedMalloc(size, kBufferAlignment));
+  uint8_t *u_data = y_data + height() * stride;
+  uint8_t *v_data = u_data + stride/2;
+
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> copy =
+    new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
+        width(), height(),
+        y_data, stride,
+        u_data, stride,
+        v_data, stride,
+        rtc::Bind(&webrtc::AlignedFree, y_data));

[magjed_webrtc, 2015/12/03 12:33:58]

...and y_data.release() here. Btw, you need to change the name |y_data| now.

[nisse-webrtc, 2015/12/03 13:35:20]

Rename, why and to what?

[magjed_webrtc, 2015/12/03 14:30:46]

Why? - Because it is not just y data anymore.
To what? - Maybe yuv_data?

+
+  ALOGE << "width: " << width() << " height: " << height() <<
+      " stride: " << stride;
+
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  ScopedLocalRefFrame local_ref_frame(jni);
+
+  ALOGE << "Looking up textureToYUV mid";
+  jmethodID transform_mid = GetMethodID(
+      jni,
+      GetObjectClass(jni, native_handle_.surface_texture_helper),
+      "textureToYUV",
+      "(Ljava/nio/ByteBuffer;IIII[F)V");
+
+  ALOGE << "Creating byte buffer";
+  jobject byte_buffer = jni->NewDirectByteBuffer(y_data, size);
+
+  /* Set u = v = 0. */
+  memset(u_data, 0x80, stride * uv_height);
+
+  // TODO(nisse): Keep java transform matrix around.
+  jfloatArray sampling_matrix = jni->NewFloatArray(16);
+  jni->SetFloatArrayRegion(sampling_matrix, 0, 16,
+                           native_handle_.sampling_matrix);
+
+  ALOGE << "Calling java textureToYUV";
+  jni->CallVoidMethod(native_handle_.surface_texture_helper,
+                      transform_mid,
+                      byte_buffer, width(), height(), stride,
+                      native_handle_.oes_texture_id, sampling_matrix);
+  CHECK_EXCEPTION(jni) << "textureToYUV throwed an exception";
+
+  return copy;
 }
 
+#else /* ! USE_CONVERT_YUV */
+rtc::scoped_refptr<webrtc::VideoFrameBuffer>
+AndroidTextureBuffer::NativeToI420Buffer() {
+  ALOGE << "NativeToI420Buffer called";
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> copy =
+    new rtc::RefCountedObject<webrtc::I420Buffer>(
+        width(), height());
+
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  ScopedLocalRefFrame local_ref_frame(jni);
+
+  ALOGE << "Looking up textureToRGBA mid";
+  jmethodID transform_mid = GetMethodID(
+      jni,
+      GetObjectClass(jni, native_handle_.surface_texture_helper),
+      "textureToRGBA",
+      "(Ljava/nio/ByteBuffer;III[F)V");
+  size_t size = 4*width() * height();
+  uint8_t *rgba = new uint8_t[size];
+  ALOGE << "Creating byte buffer";
+  jobject byte_buffer = jni->NewDirectByteBuffer(rgba, size);
+  // TODO(nisse): Keep java transform matrix around.
+  jfloatArray sampling_matrix = jni->NewFloatArray(16);
+  jni->SetFloatArrayRegion(sampling_matrix, 0, 16,
+                           native_handle_.sampling_matrix);
+
+  ALOGE << "Calling java textureToRGBA";
+  jni->CallVoidMethod(native_handle_.surface_texture_helper,
+                      transform_mid,
+                      byte_buffer, width(), height(),
+                      native_handle_.oes_texture_id, sampling_matrix);
+  CHECK_EXCEPTION(jni) << "textureToRGBA throwed an exception";
+
+  ALOGE << "Converting to YUV";
+  libyuv::ABGRToI420(
+      rgba, 4*width(),
+      copy->MutableData(webrtc::kYPlane), copy->stride(webrtc::kYPlane),
+      copy->MutableData(webrtc::kUPlane), copy->stride(webrtc::kUPlane),
+      copy->MutableData(webrtc::kVPlane), copy->stride(webrtc::kVPlane),
+      width(), height());
+
+  // TODO(nisse): Use some destructor or scoped_*, to make it
+  // exception safe.
+  delete[] rgba;
+
+  return copy;
+}
+#endif /* ! USE_CONVERT_YUV */
+
 rtc::scoped_refptr<AndroidTextureBuffer> AndroidTextureBuffer::CropAndScale(
     int cropped_input_width,
     int cropped_input_height,
     int dst_widht,
     int dst_height) {
   // TODO(perkj) Implement cropping.
   RTC_CHECK_EQ(cropped_input_width, width_);
   RTC_CHECK_EQ(cropped_input_height, height_);
 
   // Here we use Bind magic to add a reference count to |this| until the newly
   // created AndroidTextureBuffer is destructed. ScaledFrameNotInUse will be
   // called that happens and when it finishes, the reference count to |this|
   // will be decreased by one.
   return new rtc::RefCountedObject<AndroidTextureBuffer>(
       dst_widht, dst_height, native_handle_,
       rtc::KeepRefUntilDone(this));
 }
 
 }  // namespace webrtc_jni