Implement AndroidTextureBuffer::NativeToI420.

talk/app/webrtc/java/jni/native_handle_impl.cc

Index: talk/app/webrtc/java/jni/native_handle_impl.cc
diff --git a/talk/app/webrtc/java/jni/native_handle_impl.cc b/talk/app/webrtc/java/jni/native_handle_impl.cc
index 583a0380263659096660934cae8699babd5c7511..c2886d0985abfd601004dacb7870ca80ef144e04 100644
--- a/talk/app/webrtc/java/jni/native_handle_impl.cc
+++ b/talk/app/webrtc/java/jni/native_handle_impl.cc
@@ -25,23 +25,35 @@
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <android/log.h>
 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
 
+#include "talk/app/webrtc/java/jni/jni_helpers.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+#include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/keep_ref_until_done.h"
+#include "webrtc/base/logging.h"
 #include "webrtc/base/scoped_ref_ptr.h"
 
 using webrtc::NativeHandleBuffer;
 
 namespace webrtc_jni {
 
+// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD.
+static const int kBufferAlignment = 64;
+
 NativeHandleImpl::NativeHandleImpl(JNIEnv* jni,
                                    jint j_oes_texture_id,
-                                   jfloatArray j_transform_matrix)
-    : oes_texture_id(j_oes_texture_id) {
+                                   jfloatArray j_transform_matrix,
+                                   jobject helper)
+  : oes_texture_id(j_oes_texture_id),
+    surface_texture_helper(helper) {
   RTC_CHECK_EQ(16, jni->GetArrayLength(j_transform_matrix));
   jfloat* transform_matrix_ptr =
       jni->GetFloatArrayElements(j_transform_matrix, nullptr);
+  // TODO(nisse): Hold on to java float array instead, since it is

[magjed_webrtc, 2015/12/03 12:33:58]

Yes, but then you need a ScopedGlobalRef or similar. I prefer to copy the floats
for simplicity. If we want to reduce overhead, we can reduce the 4x4 matrix to
2x3.

[nisse-webrtc, 2015/12/03 13:35:20]

If we add a reference to the java array, can we delete the C++ sampling_matrix?
Or do we need both?

[magjed_webrtc, 2015/12/03 14:30:46]

Yes, then we wouldn't need the C++ sampling_matrix, but please don't add a
reference to the java array, I'm sure it will just complicate things in the
future. It's just a few bytes, like width and height, it's simpler to copy those
variables than to use global refs/reference counting.

+  // used only for callbacks into java.
   for (int i = 0; i < 16; ++i) {
     sampling_matrix[i] = transform_matrix_ptr[i];
   }
@@ -61,12 +73,117 @@ AndroidTextureBuffer::~AndroidTextureBuffer() {
   no_longer_used_cb_();
 }
 
+#define TAG "AndroidTextureBuffer"
+// Alternatively: #define AT_LOG LOG(LS_ERROR)
+#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG)
+
+#define USE_CONVERT_YUV 1
+#if USE_CONVERT_YUV
 rtc::scoped_refptr<webrtc::VideoFrameBuffer>
 AndroidTextureBuffer::NativeToI420Buffer() {
-  RTC_NOTREACHED()
-      << "AndroidTextureBuffer::NativeToI420Buffer not implemented.";
-  return nullptr;
+  ALOGE << "NativeToI420Buffer called";
+  int y_width = (width()+3) / 4;
+  int uv_width = (width()+7) / 8;
+  int stride = 8 * uv_width;
+  int uv_height = (height()+1)/2;
+  size_t size = stride * (height() + uv_height);
+  // TODO(nisse): Use scoped_ptr? The data is owned by the frame, and
+  // deleted by its destructor callback.

[magjed_webrtc, 2015/12/03 12:33:58]

Yes, use scoped_ptr here...

[nisse-webrtc, 2015/12/04 09:40:34]

Done. Not sure if it's an improvement, though. 

We allocate the buffer, then pass ownership of it to the frame. And at the same
time we also wrap it in a java ByteBuffer, which hopefully isn't used beyond the
lifetime of the pixel data. Maybe it would be cleaner to arrange so that the
data is owned by the ByteBuffer, and let the frame "own" a reference to that
buffer.

+  uint8_t *y_data = static_cast<uint8_t*>(
+      webrtc::AlignedMalloc(size, kBufferAlignment));
+  uint8_t *u_data = y_data + height() * stride;
+  uint8_t *v_data = u_data + stride/2;
+
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> copy =
+    new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
+        width(), height(),
+        y_data, stride,
+        u_data, stride,
+        v_data, stride,
+        rtc::Bind(&webrtc::AlignedFree, y_data));

[magjed_webrtc, 2015/12/03 12:33:58]

...and y_data.release() here. Btw, you need to change the name |y_data| now.

[nisse-webrtc, 2015/12/03 13:35:20]

Rename, why and to what?

[magjed_webrtc, 2015/12/03 14:30:46]

Why? - Because it is not just y data anymore.
To what? - Maybe yuv_data?

+
+  ALOGE << "width: " << width() << " height: " << height() <<
+      " stride: " << stride;
+
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  ScopedLocalRefFrame local_ref_frame(jni);
+
+  ALOGE << "Looking up textureToYUV mid";
+  jmethodID transform_mid = GetMethodID(
+      jni,
+      GetObjectClass(jni, native_handle_.surface_texture_helper),
+      "textureToYUV",
+      "(Ljava/nio/ByteBuffer;IIII[F)V");
+
+  ALOGE << "Creating byte buffer";
+  jobject byte_buffer = jni->NewDirectByteBuffer(y_data, size);
+
+  /* Set u = v = 0. */
+  memset(u_data, 0x80, stride * uv_height);
+
+  // TODO(nisse): Keep java transform matrix around.
+  jfloatArray sampling_matrix = jni->NewFloatArray(16);
+  jni->SetFloatArrayRegion(sampling_matrix, 0, 16,
+                           native_handle_.sampling_matrix);
+
+  ALOGE << "Calling java textureToYUV";
+  jni->CallVoidMethod(native_handle_.surface_texture_helper,
+                      transform_mid,
+                      byte_buffer, width(), height(), stride,
+                      native_handle_.oes_texture_id, sampling_matrix);
+  CHECK_EXCEPTION(jni) << "textureToYUV throwed an exception";
+
+  return copy;
+}
+
+#else /* ! USE_CONVERT_YUV */
+rtc::scoped_refptr<webrtc::VideoFrameBuffer>
+AndroidTextureBuffer::NativeToI420Buffer() {
+  ALOGE << "NativeToI420Buffer called";
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> copy =
+    new rtc::RefCountedObject<webrtc::I420Buffer>(
+        width(), height());
+
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  ScopedLocalRefFrame local_ref_frame(jni);
+
+  ALOGE << "Looking up textureToRGBA mid";
+  jmethodID transform_mid = GetMethodID(
+      jni,
+      GetObjectClass(jni, native_handle_.surface_texture_helper),
+      "textureToRGBA",
+      "(Ljava/nio/ByteBuffer;III[F)V");
+  size_t size = 4*width() * height();
+  uint8_t *rgba = new uint8_t[size];
+  ALOGE << "Creating byte buffer";
+  jobject byte_buffer = jni->NewDirectByteBuffer(rgba, size);
+  // TODO(nisse): Keep java transform matrix around.
+  jfloatArray sampling_matrix = jni->NewFloatArray(16);
+  jni->SetFloatArrayRegion(sampling_matrix, 0, 16,
+                           native_handle_.sampling_matrix);
+
+  ALOGE << "Calling java textureToRGBA";
+  jni->CallVoidMethod(native_handle_.surface_texture_helper,
+                      transform_mid,
+                      byte_buffer, width(), height(),
+                      native_handle_.oes_texture_id, sampling_matrix);
+  CHECK_EXCEPTION(jni) << "textureToRGBA throwed an exception";
+
+  ALOGE << "Converting to YUV";
+  libyuv::ABGRToI420(
+      rgba, 4*width(),
+      copy->MutableData(webrtc::kYPlane), copy->stride(webrtc::kYPlane),
+      copy->MutableData(webrtc::kUPlane), copy->stride(webrtc::kUPlane),
+      copy->MutableData(webrtc::kVPlane), copy->stride(webrtc::kVPlane),
+      width(), height());
+
+  // TODO(nisse): Use some destructor or scoped_*, to make it
+  // exception safe.
+  delete[] rgba;
+
+  return copy;
 }
+#endif /* ! USE_CONVERT_YUV */
 
 rtc::scoped_refptr<AndroidTextureBuffer> AndroidTextureBuffer::CropAndScale(
     int cropped_input_width,