diff --git a/Lib/fontTools/ttLib/scaleUpem.py b/Lib/fontTools/ttLib/scaleUpem.py
index 11c77f53c..93faf5c55 100644
--- a/Lib/fontTools/ttLib/scaleUpem.py
+++ b/Lib/fontTools/ttLib/scaleUpem.py
@@ -194,6 +194,9 @@ def visit(visitor, obj, attr, cff):
             else:
                 setattr(topDict, attr, visitor.scale(value))
 
+        topDict.FontMatrix[0] /= visitor.scaleFactor
+        topDict.FontMatrix[3] /= visitor.scaleFactor
+
         for private in privates:
             for attr in (
                 "BlueValues",