[reactos] 01/01: [WIN32SS][NTGDI] Strictly check in XFORMOBJ_iSetXform (#1676)

[Katayama Hirofumi MZ]

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=bf06c94e15c47e549e824…

commit bf06c94e15c47e549e824a1f82e672d40b150bc6
Author:     Katayama Hirofumi MZ <katayama.hirofumi.mz(a)gmail.com>
AuthorDate: Mon Jun 24 21:32:40 2019 +0900
Commit:     GitHub <noreply(a)github.com>
CommitDate: Mon Jun 24 21:32:40 2019 +0900

    [WIN32SS][NTGDI] Strictly check in XFORMOBJ_iSetXform (#1676)
    
    Check if the determinant of the given transformation matrix is non-zero. If zero, the
function returns DDI_ERROR without change. CORE-15554
---
 win32ss/gdi/ntgdi/xformobj.c | 60 ++++++++++++++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/win32ss/gdi/ntgdi/xformobj.c b/win32ss/gdi/ntgdi/xformobj.c
index fccf5517b38..ebd8f6b19ef 100644
--- a/win32ss/gdi/ntgdi/xformobj.c
+++ b/win32ss/gdi/ntgdi/xformobj.c
@@ -104,12 +104,9 @@ HintFromAccel(ULONG flAccel)
 /** Internal functions ********************************************************/
 
 ULONG
-NTAPI
-XFORMOBJ_UpdateAccel(
-    IN OUT XFORMOBJ *pxo)
+FASTCALL
+MX_UpdateAccel(IN OUT PMATRIX pmx)
 {
-    PMATRIX pmx = XFORMOBJ_pmx(pxo);
-
     /* Copy Dx and Dy to FIX format */
     pmx->fxDx = FLOATOBJ_GetFix(&pmx->efDx);
     pmx->fxDy = FLOATOBJ_GetFix(&pmx->efDy);
@@ -143,6 +140,16 @@ XFORMOBJ_UpdateAccel(
     return HintFromAccel(pmx->flAccel);
 }
 
+ULONG
+NTAPI
+XFORMOBJ_UpdateAccel(
+    IN OUT XFORMOBJ *pxo)
+{
+    PMATRIX pmx = XFORMOBJ_pmx(pxo);
+
+    return MX_UpdateAccel(pmx);
+}
+
 
 ULONG
 NTAPI
@@ -150,24 +157,45 @@ XFORMOBJ_iSetXform(
     IN OUT XFORMOBJ *pxo,
     IN const XFORML *pxform)
 {
-    PMATRIX pmx = XFORMOBJ_pmx(pxo);
+    PMATRIX pmx;
+    MATRIX mxTemp;
+    ULONG Hint;
 
     /* Check parameters */
     if (!pxo || !pxform) return DDI_ERROR;
 
-    /* Check if the xform is valid */
-    if ((pxform->eM11 == 0) || (pxform->eM22 == 0)) return DDI_ERROR;
-
     /* Copy members */
-    FLOATOBJ_SetFloat(&pmx->efM11, pxform->eM11);
-    FLOATOBJ_SetFloat(&pmx->efM12, pxform->eM12);
-    FLOATOBJ_SetFloat(&pmx->efM21, pxform->eM21);
-    FLOATOBJ_SetFloat(&pmx->efM22, pxform->eM22);
-    FLOATOBJ_SetFloat(&pmx->efDx, pxform->eDx);
-    FLOATOBJ_SetFloat(&pmx->efDy, pxform->eDy);
+    FLOATOBJ_SetFloat(&mxTemp.efM11, pxform->eM11);
+    FLOATOBJ_SetFloat(&mxTemp.efM12, pxform->eM12);
+    FLOATOBJ_SetFloat(&mxTemp.efM21, pxform->eM21);
+    FLOATOBJ_SetFloat(&mxTemp.efM22, pxform->eM22);
+    FLOATOBJ_SetFloat(&mxTemp.efDx, pxform->eDx);
+    FLOATOBJ_SetFloat(&mxTemp.efDy, pxform->eDy);
 
     /* Update accelerators and return complexity */
-    return XFORMOBJ_UpdateAccel(pxo);
+    Hint = MX_UpdateAccel(&mxTemp);
+
+    /* Check whether det = (M11 * M22 - M12 * M21) is non-zero */
+    if (Hint == GX_SCALE)
+    {
+        if (FLOATOBJ_Equal0(&mxTemp.efM11) || FLOATOBJ_Equal0(&mxTemp.efM22))
+        {
+            return DDI_ERROR;
+        }
+    }
+    else if (Hint == GX_GENERAL)
+    {
+        if (!MX_IsInvertible(&mxTemp))
+        {
+            return DDI_ERROR;
+        }
+    }
+
+    /* Store */
+    pmx = XFORMOBJ_pmx(pxo);
+    *pmx = mxTemp;
+
+    return Hint;
 }