[cffLib] Fix for #1451 (#1456)

* a charstring is not guaranteed to end in an operator, so the final bytecodes 11 and 14 can be part of an encoded numeric value; so remove 'return' or 'endchar' at the program level instead of bytescode
* move non-CFF2 test+error to elif clause of earlier isCFF2 test
Fixes the remaining issue from #1451

Lib/fontTools/misc/psCharStrings.py

@@ -983,6 +983,16 @@ class T2CharString(object):
 			return
 		opcodes = self.opcodes
 		program = self.program
+
+		if isCFF2:
+			# If present, remove return and endchar operators.
+			if program and program[-1] in ("return", "endchar"):
+				program = program[:-1]
+		elif program and not isinstance(program[-1], basestring):
+			raise CharStringCompileError(
+				"T2CharString or Subr has items on the stack after last operator."
+			)
+
 		bytecode = []
 		encodeInt = self.getIntEncoder()
 		encodeFixed = self.getFixedEncoder()
@@ -1005,10 +1015,6 @@ class T2CharString(object):
 				bytecode.append(encodeFixed(token))
 			else:
 				assert 0, "unsupported type: %s" % type(token)
-		if not isCFF2 and program and not isinstance(program[-1], basestring):
-			raise CharStringCompileError(
-				"T2CharString or Subr has items on the stack after last operator."
-			)
 		try:
 			bytecode = bytesjoin(bytecode)
 		except TypeError:
@@ -1016,11 +1022,6 @@ class T2CharString(object):
 			raise
 		self.setBytecode(bytecode)
 
-		if isCFF2:
-			# If present, remove return and endchar operators.
-			if self.bytecode and (byteord(self.bytecode[-1]) in (11, 14)):
-				self.bytecode = self.bytecode[:-1]
-
 	def needsDecompilation(self):
 		return self.bytecode is not None