Defer vblank event handling while waiting for a pending flip

This is to avoid submitting more flips while we are waiting for pending
ones to complete.

(Ported from amdgpu commit e52872da69ecc84dafb3355839e35b0383f0d228)
Acked-by: Alex Deucher <alexander.deucher@amd.com>

src/drmmode_display.c

@@ -324,6 +324,9 @@ drmmode_do_crtc_dpms(xf86CrtcPtr crtc, int mode)
 				nominal_frame_rate /= pix_in_frame;
 			drmmode_crtc->dpms_last_fps = nominal_frame_rate;
 		}
+
+		drmmode_crtc->dpms_mode = mode;
+		radeon_drm_queue_handle_deferred(crtc);
 	} else if (drmmode_crtc->dpms_mode != DPMSModeOn && mode == DPMSModeOn) {
 		/*
 		 * Off->On transition: calculate and accumulate the
@@ -341,8 +344,9 @@ drmmode_do_crtc_dpms(xf86CrtcPtr crtc, int mode)
 			drmmode_crtc->interpolated_vblanks += delta_seq;
 
 		}
+
+		drmmode_crtc->dpms_mode = DPMSModeOn;
 	}
-	drmmode_crtc->dpms_mode = mode;
 }
 
 static void
@@ -972,6 +976,7 @@ done:
 		}
 	}
 
+	radeon_drm_queue_handle_deferred(crtc);
 	return ret;
 }
 
@@ -1763,11 +1768,6 @@ drmmode_output_set_tear_free(RADEONEntPtr pRADEONEnt,
 	drmmode_output->tear_free = tear_free;
 
 	if (crtc) {
-		/* Wait for pending flips before drmmode_set_mode_major calls
-		 * drmmode_crtc_update_tear_free, to prevent a nested
-		 * drmHandleEvent call, which would hang
-		 */
-		radeon_drm_wait_pending_flip(crtc);
 		drmmode_set_mode_major(crtc, &crtc->mode, crtc->rotation,
 				       crtc->x, crtc->y);
 	}
@@ -3278,6 +3278,7 @@ Bool radeon_do_pageflip(ScrnInfoPtr scrn, ClientPtr client,
 	drmmode_crtc_private_ptr drmmode_crtc = config->crtc[0]->driver_private;
 	uint32_t flip_flags = flip_sync == FLIP_ASYNC ? DRM_MODE_PAGE_FLIP_ASYNC : 0;
 	drmmode_flipdata_ptr flipdata;
+	Bool handle_deferred = FALSE;
 	uintptr_t drm_queue_seq = 0;
 	struct drmmode_fb *fb;
 	int i = 0;
@@ -3360,6 +3361,7 @@ Bool radeon_do_pageflip(ScrnInfoPtr scrn, ClientPtr client,
 
 			if (drmmode_crtc->scanout_update_pending) {
 				radeon_drm_wait_pending_flip(crtc);
+				handle_deferred = TRUE;
 				radeon_drm_abort_entry(drmmode_crtc->scanout_update_pending);
 				drmmode_crtc->scanout_update_pending = 0;
 			}
@@ -3395,6 +3397,8 @@ Bool radeon_do_pageflip(ScrnInfoPtr scrn, ClientPtr client,
 		drm_queue_seq = 0;
 	}
 
+	if (handle_deferred)
+		radeon_drm_queue_handle_deferred(ref_crtc);
 	if (flipdata->flip_count > 0)
 		return TRUE;
 
@@ -3414,5 +3418,7 @@ error:
 
 	xf86DrvMsg(scrn->scrnIndex, X_WARNING, "Page flip failed: %s\n",
 		   strerror(errno));
+	if (handle_deferred)
+		radeon_drm_queue_handle_deferred(ref_crtc);
 	return FALSE;
 }

src/drmmode_display.h

@@ -103,6 +103,10 @@ typedef struct {
      * modeset)
      */
     Bool need_modeset;
+    /* For keeping track of nested calls to drm_wait_pending_flip /
+     * drm_queue_handle_deferred
+     */
+    int wait_flip_nesting_level;
     /* A flip to this FB is pending for this CRTC */
     struct drmmode_fb *flip_pending;
     /* The FB currently being scanned out by this CRTC, if any */

src/radeon_drm_queue.c

@@ -117,6 +117,30 @@ radeon_drm_vblank_handler(int fd, unsigned int frame, unsigned int sec,
 			     user_ptr);
 }
 
+/*
+ * Handle deferred DRM vblank events
+ *
+ * This function must be called after radeon_drm_wait_pending_flip, once
+ * it's safe to attempt queueing a flip again
+ */
+void
+radeon_drm_queue_handle_deferred(xf86CrtcPtr crtc)
+{
+    drmmode_crtc_private_ptr drmmode_crtc = crtc->driver_private;
+    struct radeon_drm_queue_entry *e, *tmp;
+
+    if (drmmode_crtc->wait_flip_nesting_level == 0 ||
+	--drmmode_crtc->wait_flip_nesting_level > 0)
+	return;
+
+    xorg_list_for_each_entry_safe(e, tmp, &radeon_drm_vblank_signalled, list) {
+	drmmode_crtc_private_ptr drmmode_crtc = e->crtc->driver_private;
+
+	if (drmmode_crtc->wait_flip_nesting_level == 0)
+	    radeon_drm_queue_handle_one(e);
+    }
+}
+
 /*
  * Enqueue a potential drm response; when the associated response
  * appears, we've got data to pass to the handler from here
@@ -191,6 +215,13 @@ radeon_drm_abort_entry(uintptr_t seq)
     if (seq == RADEON_DRM_QUEUE_ERROR)
 	return;
 
+    xorg_list_for_each_entry_safe(e, tmp, &radeon_drm_vblank_signalled, list) {
+	if (e->seq == seq) {
+	    radeon_drm_abort_one(e);
+	    return;
+	}
+    }
+
     xorg_list_for_each_entry_safe(e, tmp, &radeon_drm_queue, list) {
 	if (e->seq == seq) {
 	    radeon_drm_abort_one(e);
@@ -229,8 +260,12 @@ radeon_drm_handle_event(int fd, drmEventContext *event_context)
     xorg_list_for_each_entry_safe(e, tmp, &radeon_drm_flip_signalled, list)
 	radeon_drm_queue_handle_one(e);
 
-    xorg_list_for_each_entry_safe(e, tmp, &radeon_drm_vblank_signalled, list)
-	radeon_drm_queue_handle_one(e);
+    xorg_list_for_each_entry_safe(e, tmp, &radeon_drm_vblank_signalled, list) {
+	drmmode_crtc_private_ptr drmmode_crtc = e->crtc->driver_private;
+
+	if (drmmode_crtc->wait_flip_nesting_level == 0)
+	    radeon_drm_queue_handle_one(e);
+    }
 
     return r;
 }
@@ -244,6 +279,8 @@ void radeon_drm_wait_pending_flip(xf86CrtcPtr crtc)
     RADEONEntPtr pRADEONEnt = RADEONEntPriv(crtc->scrn);
     struct radeon_drm_queue_entry *e, *tmp;
 
+    drmmode_crtc->wait_flip_nesting_level++;
+
     xorg_list_for_each_entry_safe(e, tmp, &radeon_drm_flip_signalled, list)
 	radeon_drm_queue_handle_one(e);
 

src/radeon_drm_queue.h

@@ -40,6 +40,7 @@ typedef void (*radeon_drm_handler_proc)(xf86CrtcPtr crtc, uint32_t seq,
 					uint64_t usec, void *data);
 typedef void (*radeon_drm_abort_proc)(xf86CrtcPtr crtc, void *data);
 
+void radeon_drm_queue_handle_deferred(xf86CrtcPtr crtc);
 uintptr_t radeon_drm_queue_alloc(xf86CrtcPtr crtc, ClientPtr client,
 				 uint64_t id, void *data,
 				 radeon_drm_handler_proc handler,