Solaris: Avoid switching to inactive VT's

Fix for OpenSolaris bug 6876992: "[vconsole] Ctrl+Alt+F12 switchs to blank
console screen with hotkeys property turned-off"
http://bugs.opensolaris.org/bugdatabase/view_bug.do?bug_id=6876992

Xorg needs to do sanity test for the VT it is commanded to switch to.
If the VT is not opened by any process, discard the switching request.

The changes also contain the fix for some flaws discovered when
getting the new gdm to run.

Signed-off-by: Aaron Zang <Aaron.Zang@Sun.COM>
Signed-off-by: Alan Coopersmith <alan.coopersmith@sun.com>

3 files changed, 43 insertions, 12 deletions

diff --git a/hw/xfree86/common/xf86Events.c b/hw/xfree86/common/xf86Events.c
index 8cd765a1c..8e6a15be8 100644
--- a/hw/xfree86/common/xf86Events.c
+++ b/hw/xfree86/common/xf86Events.c
@@ -202,8 +202,16 @@ xf86ProcessActionEvent(ActionEvent action, void *arg)
 	    vtno--;
 #endif
 #if defined(sun)
-	    if (vtno == xf86Info.vtno)
+	    if (vtno == xf86Info.vtno) {
 		break;
+	    } else {
+		struct vt_stat state;
+		if (ioctl(xf86Info.consoleFd, VT_GETSTATE, &state) < 0)
+			break;
+
+		if ((state.v_state & (1 << vtno)) == 0)
+			break;
+	    }
 
 	    xf86Info.vtRequestsPending = TRUE;
 	    xf86Info.vtPendingNum = vtno;
diff --git a/hw/xfree86/os-support/solaris/sun_VTsw.c b/hw/xfree86/os-support/solaris/sun_VTsw.c
index ded2f271e..7f4e08e1e 100644
--- a/hw/xfree86/os-support/solaris/sun_VTsw.c
+++ b/hw/xfree86/os-support/solaris/sun_VTsw.c
@@ -38,17 +38,27 @@
  * Handle the VT-switching interface for Solaris/OpenSolaris
  */
 
+static int xf86VTPruneDoor = 0;
+
 void
-xf86VTRequest(int sig)
+xf86VTRelease(int sig)
 {
-	if (xf86Info.vtPendingNum != -1)
+	if (xf86Info.vtPendingNum == -1)
 	{
-		ioctl(xf86Info.consoleFd, VT_RELDISP, 1);
-		xf86Info.vtPendingNum = -1;
-
+		xf86VTPruneDoor = 1;
+		xf86Info.vtRequestsPending = TRUE;
 		return;
 	}
 
+	ioctl(xf86Info.consoleFd, VT_RELDISP, 1);
+	xf86Info.vtPendingNum = -1;
+
+	return;
+}
+
+void
+xf86VTAcquire(int sig)
+{
 	xf86Info.vtRequestsPending = TRUE;
 	return;
 }
@@ -68,6 +78,12 @@ xf86VTSwitchAway(void)
 
 	xf86Info.vtRequestsPending = FALSE;
 
+	if (xf86VTPruneDoor) {
+		xf86VTPruneDoor = 0;
+		ioctl(xf86Info.consoleFd, VT_RELDISP, 1);
+		return (TRUE);
+	}
+
 	vt_door_arg.vt_ev = VT_EV_HOTKEYS;
 	vt_door_arg.vt_num = xf86Info.vtPendingNum;
 	door_arg.data_ptr = (char *)&vt_door_arg;
diff --git a/hw/xfree86/os-support/solaris/sun_init.c b/hw/xfree86/os-support/solaris/sun_init.c
index 2c569f02c..5846866a2 100644
--- a/hw/xfree86/os-support/solaris/sun_init.c
+++ b/hw/xfree86/os-support/solaris/sun_init.c
@@ -39,6 +39,8 @@ static Bool Protect0 = FALSE;
 static int VTnum = -1;
 static int xf86StartVT = -1;
 static int vtEnabled = 0;
+extern void xf86VTAcquire(int);
+extern void xf86VTRelease(int);
 #endif
 
 /* Device to open as xf86Info.consoleFd */
@@ -137,7 +139,8 @@ xf86OpenConsole(void)
 	    else
 	    {
 		if ((ioctl(fd, VT_OPENQRY, &xf86Info.vtno) < 0) ||
-		    (xf86Info.vtno == -1)) {
+		    (xf86Info.vtno == -1))
+		{
 		    FatalError("xf86OpenConsole: Cannot find a free VT\n");
 		}
 	    }
@@ -146,7 +149,8 @@ xf86OpenConsole(void)
 	    snprintf(consoleDev, PATH_MAX, "/dev/vt/%d", xf86Info.vtno);
 	}
 
-	if (fd != -1) {
+	if (fd != -1)
+	{
 	    close(fd);
 	}
 
@@ -178,11 +182,12 @@ xf86OpenConsole(void)
 	    if (ioctl(xf86Info.consoleFd, VT_GETMODE, &VT) < 0)
 		FatalError("xf86OpenConsole: VT_GETMODE failed\n");
 
-	    OsSignal(SIGUSR1, xf86VTRequest);
+	    OsSignal(SIGUSR1, xf86VTAcquire);
+	    OsSignal(SIGUSR2, xf86VTRelease);
 
 	    VT.mode = VT_PROCESS;
-	    VT.relsig = SIGUSR1;
 	    VT.acqsig = SIGUSR1;
+	    VT.relsig = SIGUSR2;
 
 	    if (ioctl(xf86Info.consoleFd, VT_SETMODE, &VT) < 0)
 		FatalError("xf86OpenConsole: VT_SETMODE VT_PROCESS failed\n");
@@ -204,7 +209,8 @@ xf86OpenConsole(void)
     else /* serverGeneration != 1 */
     {
 #ifdef HAS_USL_VTS
-	if (vtEnabled) {
+	if (vtEnabled)
+	{
 	    /*
 	     * Now re-get the VT
 	     */
@@ -285,7 +291,8 @@ xf86CloseConsole(void)
 #endif
 
 #ifdef HAS_USL_VTS
-    if (vtEnabled == 1) {
+    if (vtEnabled)
+    {
 	if (ioctl(xf86Info.consoleFd, VT_GETMODE, &VT) != -1)
 	{
 	    VT.mode = VT_AUTO;		/* Set default vt handling */