Update XServerPixelBuffer to handle shmget() errors properly.

webrtc/modules/desktop_capture/x11/x_server_pixel_buffer.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 50 matching lines @@
 
 XServerPixelBuffer::XServerPixelBuffer() {}
 
 XServerPixelBuffer::~XServerPixelBuffer() {
   Release();
 }
 
 void XServerPixelBuffer::Release() {
   if (x_image_) {
     XDestroyImage(x_image_);
-    x_image_ = NULL;
+    x_image_ = nullptr;
   }
   if (shm_pixmap_) {
     XFreePixmap(display_, shm_pixmap_);
     shm_pixmap_ = 0;
   }
   if (shm_gc_) {
     XFreeGC(display_, shm_gc_);
-    shm_gc_ = NULL;
+    shm_gc_ = nullptr;
   }
-  if (shm_segment_info_) {
-    if (shm_segment_info_->shmaddr != reinterpret_cast<char*>(-1))
-      shmdt(shm_segment_info_->shmaddr);
-    if (shm_segment_info_->shmid != -1)
-      shmctl(shm_segment_info_->shmid, IPC_RMID, 0);
-    delete shm_segment_info_;
-    shm_segment_info_ = NULL;
-  }
+
+  ReleaseSharedMemorySegment();
+
   window_ = 0;
 }
 
+void XServerPixelBuffer::ReleaseSharedMemorySegment() {
+  if (!shm_segment_info_)
+    return;
+  if (shm_segment_info_->shmaddr != nullptr)
+    shmdt(shm_segment_info_->shmaddr);
+  if (shm_segment_info_->shmid != -1)
+    shmctl(shm_segment_info_->shmid, IPC_RMID, 0);
+  delete shm_segment_info_;
+  shm_segment_info_ = nullptr;
+}
+
 bool XServerPixelBuffer::Init(Display* display, Window window) {
   Release();
   display_ = display;
 
   XWindowAttributes attributes;
   {
     XErrorTrap error_trap(display_);
     if (!XGetWindowAttributes(display_, window, &attributes) ||
         error_trap.GetLastErrorAndDisable() != 0) {
       return false;
@@ skipping 14 matching lines @@
   int major, minor;
   Bool have_pixmaps;
   if (!XShmQueryVersion(display_, &major, &minor, &have_pixmaps)) {
     // Shared memory not supported. CaptureRect will use the XImage API instead.
     return;
   }
 
   bool using_shm = false;
   shm_segment_info_ = new XShmSegmentInfo;
   shm_segment_info_->shmid = -1;
-  shm_segment_info_->shmaddr = reinterpret_cast<char*>(-1);
+  shm_segment_info_->shmaddr = nullptr;
   shm_segment_info_->readOnly = False;
   x_image_ = XShmCreateImage(display_, default_visual, default_depth, ZPixmap,
                              0, shm_segment_info_, window_size_.width(),
                              window_size_.height());
   if (x_image_) {
-    shm_segment_info_->shmid = shmget(
-        IPC_PRIVATE, x_image_->bytes_per_line * x_image_->height,
-        IPC_CREAT | 0600);
+    shm_segment_info_->shmid =
+        shmget(IPC_PRIVATE, x_image_->bytes_per_line * x_image_->height,
+               IPC_CREAT | 0600);
     if (shm_segment_info_->shmid != -1) {
-      shm_segment_info_->shmaddr = x_image_->data =
-          reinterpret_cast<char*>(shmat(shm_segment_info_->shmid, 0, 0));
-      if (x_image_->data != reinterpret_cast<char*>(-1)) {
+      void* shmat_result = shmat(shm_segment_info_->shmid, 0, 0);
+      if (shmat_result != reinterpret_cast<void*>(-1)) {
+        shm_segment_info_->shmaddr = reinterpret_cast<char*>(shmat_result);
+        x_image_->data = shm_segment_info_->shmaddr;
+
         XErrorTrap error_trap(display_);
         using_shm = XShmAttach(display_, shm_segment_info_);
         XSync(display_, False);
         if (error_trap.GetLastErrorAndDisable() != 0)
           using_shm = false;
         if (using_shm) {
           LOG(LS_VERBOSE) << "Using X shared memory segment "
                           << shm_segment_info_->shmid;
         }
       }
     } else {
       LOG(LS_WARNING) << "Failed to get shared memory segment. "
                       "Performance may be degraded.";
     }
   }
 
   if (!using_shm) {
     LOG(LS_WARNING) << "Not using shared memory. Performance may be degraded.";
-    Release();
+    ReleaseSharedMemorySegment();
     return;
   }
 
   if (have_pixmaps)
     have_pixmaps = InitPixmaps(default_depth);
 
   shmctl(shm_segment_info_->shmid, IPC_RMID, 0);
   shm_segment_info_->shmid = -1;
 
   LOG(LS_VERBOSE) << "Using X shared memory extension v"
@@ skipping 163 matching lines @@
       // Write as 32-bit RGB.
       dst_pos_32[x] = ((r >> 8) & 0xff0000) | ((g >> 16) & 0xff00) |
           ((b >> 24) & 0xff);
     }
     dst_pos += frame->stride();
     src_pos += src_stride;
   }
 }
 
 }  // namespace webrtc