hwcontext_vaapi: Try to create devices via DRM before X11

Opening the device via X11 (DRI2/DRI3) rather than opening a DRM render
node directly is only useful if you intend to use the legacy X11 interop
functions.  That's never true for the ffmpeg utility, and a library user
who does want this will likely provide their own display instance rather
than making a new one here.

libavutil/hwcontext_vaapi.c

@@ -1500,27 +1500,6 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         try_x11 = HAVE_VAAPI_X11;
     }
 
-#if HAVE_VAAPI_X11
-    if (!display && try_x11) {
-        // Try to open the device as an X11 display.
-        priv->x11_display = XOpenDisplay(device);
-        if (!priv->x11_display) {
-            av_log(ctx, AV_LOG_VERBOSE, "Cannot open X11 display "
-                   "%s.\n", XDisplayName(device));
-        } else {
-            display = vaGetDisplay(priv->x11_display);
-            if (!display) {
-                av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
-                       "from X11 display %s.\n", XDisplayName(device));
-                return AVERROR_UNKNOWN;
-            }
-
-            av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
-                   "X11 display %s.\n", XDisplayName(device));
-        }
-    }
-#endif
-
 #if HAVE_VAAPI_DRM
     while (!display && try_drm) {
         // If the device is specified, try to open it as a DRM device node.
@@ -1588,6 +1567,27 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     }
 #endif
 
+#if HAVE_VAAPI_X11
+    if (!display && try_x11) {
+        // Try to open the device as an X11 display.
+        priv->x11_display = XOpenDisplay(device);
+        if (!priv->x11_display) {
+            av_log(ctx, AV_LOG_VERBOSE, "Cannot open X11 display "
+                   "%s.\n", XDisplayName(device));
+        } else {
+            display = vaGetDisplay(priv->x11_display);
+            if (!display) {
+                av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
+                       "from X11 display %s.\n", XDisplayName(device));
+                return AVERROR_UNKNOWN;
+            }
+
+            av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
+                   "X11 display %s.\n", XDisplayName(device));
+        }
+    }
+#endif
+
     if (!display) {
         if (device)
             av_log(ctx, AV_LOG_ERROR, "No VA display found for "