[hbelusca] 59746: [NTVDM] Do not rely on the internal format for console handles. Instead, use a more reliable way, by checking for success when calling a simple console function acting on in/out c...

Author: hbelusca
Date: Thu Aug 15 21:10:09 2013
New Revision: 59746

URL: http://svn.reactos.org/svn/reactos?rev=59746&view=rev
Log:
[NTVDM]
Do not rely on the internal format for console handles. Instead, use a more reliable way,
by checking for success when calling a simple console function acting on in/out console
handles.

Modified:
    branches/ntvdm/subsystems/ntvdm/dos.c
    branches/ntvdm/subsystems/ntvdm/ntvdm.h

Modified: branches/ntvdm/subsystems/ntvdm/dos.c
URL:
http://svn.reactos.org/svn/reactos/branches/ntvdm/subsystems/ntvdm/dos.c?re…
==============================================================================
--- branches/ntvdm/subsystems/ntvdm/dos.c	[iso-8859-1] (original)
+++ branches/ntvdm/subsystems/ntvdm/dos.c	[iso-8859-1] Thu Aug 15 21:10:09 2013
@@ -28,6 +28,30 @@
 static BOOLEAN DosUmbLinked = FALSE;
 
 /* PRIVATE FUNCTIONS **********************************************************/
+
+/* Taken from base/shell/cmd/console.c */
+static BOOL IsConsoleHandle(HANDLE hHandle)
+{
+    DWORD dwMode;
+
+    /* Check whether the handle may be that of a console... */
+    if ((GetFileType(hHandle) & FILE_TYPE_CHAR) == 0) return FALSE;
+
+    /*
+     * It may be. Perform another test... The idea comes from the
+     * MSDN description of the WriteConsole API:
+     *
+     * "WriteConsole fails if it is used with a standard handle
+     *  that is redirected to a file. If an application processes
+     *  multilingual output that can be redirected, determine whether
+     *  the output handle is a console handle (one method is to call
+     *  the GetConsoleMode function and check whether it succeeds).
+     *  If the handle is a console handle, call WriteConsole. If the
+     *  handle is not a console handle, the output is redirected and
+     *  you should call WriteFile to perform the I/O."
+     */
+    return GetConsoleMode(hHandle, &dwMode);
+}
 
 static VOID DosCombineFreeBlocks(WORD StartBlock)
 {

Modified: branches/ntvdm/subsystems/ntvdm/ntvdm.h
URL:
http://svn.reactos.org/svn/reactos/branches/ntvdm/subsystems/ntvdm/ntvdm.h?…
==============================================================================
--- branches/ntvdm/subsystems/ntvdm/ntvdm.h	[iso-8859-1] (original)
+++ branches/ntvdm/subsystems/ntvdm/ntvdm.h	[iso-8859-1] Thu Aug 15 21:10:09 2013
@@ -26,7 +26,6 @@
 #define MAX_ADDRESS TO_LINEAR(MAX_SEGMENT, MAX_OFFSET)
 #define FAR_POINTER(x) ((ULONG_PTR)BaseAddress + TO_LINEAR(HIWORD(x), LOWORD(x)))
 #define STEPS_PER_CYCLE 256
-#define IsConsoleHandle(h) (((((ULONG_PTR)h) & 0x10000003) == 3) ? TRUE : FALSE)
 
 /* FUNCTIONS ******************************************************************/