From df0d6bbadfb6f8e1f468b6d54e442cae76c4fdfe Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 15:26:32 -0600 Subject: [PATCH 01/29] [otBase] Add array version of various int readers --- Lib/fontTools/ttLib/tables/otBase.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index 405f1f868..b2b8455bd 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -140,10 +140,6 @@ class OTTableReader(object): value, = struct.unpack(f">{typecode}", self.data[pos:newpos]) self.pos = newpos return value - - def readUShort(self): - return self.readValue("H", staticSize=2) - def readArray(self, typecode, staticSize, count): pos = self.pos newpos = pos + count * staticSize @@ -152,20 +148,35 @@ class OTTableReader(object): self.pos = newpos return value - def readUShortArray(self, count): - return self.readArray("H", staticSize=2, count=count) - def readInt8(self): return self.readValue("b", staticSize=1) + def readInt8Array(self, count): + return self.readArray("b", staticSize=1, count=count) def readShort(self): return self.readValue("h", staticSize=2) + def readShortArray(self, count): + return self.readArray("h", staticSize=2, count=count) def readLong(self): return self.readValue("l", staticSize=4) + def readLongArray(self, count): + return self.readArray("l", staticSize=4, count=count) def readUInt8(self): return self.readValue("B", staticSize=1) + def readUInt8Array(self, count): + return self.readArray("B", staticSize=1, count=count) + + def readUShort(self): + return self.readValue("H", staticSize=2) + def readUShortArray(self, count): + return self.readArray("H", staticSize=2, count=count) + + def readULong(self): + return self.readValue("L", staticSize=4) + def readULongArray(self, count): + return self.readArray("L", staticSize=4, count=count) def readUInt24(self): pos = self.pos @@ -174,8 +185,6 @@ class OTTableReader(object): self.pos = newpos return value - def readULong(self): - return self.readValue("L", staticSize=4) def readTag(self): pos = self.pos From 1163fe68ffaebdd49ec04b24cce5391657f872a5 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 15:26:49 -0600 Subject: [PATCH 02/29] [otConverters] Use array readers when reading ItemVariationStore payload --- Lib/fontTools/ttLib/tables/otConverters.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index 4af38acd1..a137f38a9 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -1596,12 +1596,10 @@ class VarDataValue(BaseConverter): regionCount = tableDict["VarRegionCount"] shortCount = tableDict["NumShorts"] - for i in range(min(regionCount, shortCount)): - values.append(reader.readShort()) - for i in range(min(regionCount, shortCount), regionCount): - values.append(reader.readInt8()) - for i in range(regionCount, shortCount): - reader.readInt8() + n1, n2 = min(regionCount, shortCount), max(regionCount, shortCount) + values.extend(reader.readShortArray(n1)) + values.extend(reader.readInt8Array(n2 - n1)) + del values[regionCount:] return values From 2496dcf9cc0e6966379f097307a6ee2b874e44f6 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 15:41:25 -0600 Subject: [PATCH 03/29] [otConverters] Add array readers to int converters --- Lib/fontTools/ttLib/tables/otConverters.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index a137f38a9..c443f1aa0 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -230,6 +230,8 @@ class Long(IntValue): staticSize = 4 def read(self, reader, font, tableDict): return reader.readLong() + def readArray(self, reader, font, tableDict, count): + return reader.readLongArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeLong(value) @@ -237,6 +239,8 @@ class ULong(IntValue): staticSize = 4 def read(self, reader, font, tableDict): return reader.readULong() + def readArray(self, reader, font, tableDict, count): + return reader.readULongArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeULong(value) @@ -249,6 +253,8 @@ class Short(IntValue): staticSize = 2 def read(self, reader, font, tableDict): return reader.readShort() + def readArray(self, reader, font, tableDict, count): + return reader.readShortArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeShort(value) @@ -256,6 +262,8 @@ class UShort(IntValue): staticSize = 2 def read(self, reader, font, tableDict): return reader.readUShort() + def readArray(self, reader, font, tableDict, count): + return reader.readUShortArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeUShort(value) @@ -263,6 +271,8 @@ class Int8(IntValue): staticSize = 1 def read(self, reader, font, tableDict): return reader.readInt8() + def readArray(self, reader, font, tableDict, count): + return reader.readInt8Array(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeInt8(value) @@ -270,6 +280,8 @@ class UInt8(IntValue): staticSize = 1 def read(self, reader, font, tableDict): return reader.readUInt8() + def readArray(self, reader, font, tableDict, count): + return reader.readUInt8Array(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeUInt8(value) From 1fd2a44bbf15e2902f95072b3fc95af8309f16a8 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 15:57:53 -0600 Subject: [PATCH 04/29] [otBase] fix array-reader to return list, not array.array Was not noticed because it was for the most part unused. --- Lib/fontTools/ttLib/tables/otBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index b2b8455bd..31ff2d75f 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -146,7 +146,7 @@ class OTTableReader(object): value = array.array(typecode, self.data[pos:newpos]) if sys.byteorder != "big": value.byteswap() self.pos = newpos - return value + return value.tolist() def readInt8(self): return self.readValue("b", staticSize=1) From 36dd271cd5d22692226bce23081a3b26a27323e1 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 15:58:45 -0600 Subject: [PATCH 05/29] [otBase/otConverters] Add array-writers for int types --- Lib/fontTools/ttLib/tables/otBase.py | 12 ++++++++++++ Lib/fontTools/ttLib/tables/otConverters.py | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index 31ff2d75f..facef94fe 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -428,22 +428,34 @@ class OTTableWriter(object): def writeValue(self, typecode, value): self.items.append(struct.pack(f">{typecode}", value)) + def writeArray(self, typecode, values): + a = array.array(typecode, values) + if sys.byteorder != "big": a.byteswap() + self.items.append(a.tobytes()) def writeUShort(self, value): assert 0 <= value < 0x10000, value self.items.append(struct.pack(">H", value)) + def writeUShortArray(self, values): + self.writeArray('H', values) def writeShort(self, value): assert -32768 <= value < 32768, value self.items.append(struct.pack(">h", value)) + def writeShortArray(self, values): + self.writeArray('h', values) def writeUInt8(self, value): assert 0 <= value < 256, value self.items.append(struct.pack(">B", value)) + def writeUInt8Array(self, values): + self.writeArray('B', values) def writeInt8(self, value): assert -128 <= value < 128, value self.items.append(struct.pack(">b", value)) + def writeInt8Array(self, values): + self.writeArray('b', values) def writeUInt24(self, value): assert 0 <= value < 0x1000000, value diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index c443f1aa0..ab2037f00 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -234,6 +234,8 @@ class Long(IntValue): return reader.readLongArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeLong(value) + def writeArray(self, writer, font, tableDict, values): + writer.writeLongArray(values) class ULong(IntValue): staticSize = 4 @@ -243,6 +245,8 @@ class ULong(IntValue): return reader.readULongArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeULong(value) + def writeArray(self, writer, font, tableDict, values): + writer.writeULongArray(values) class Flags32(ULong): @staticmethod @@ -257,6 +261,8 @@ class Short(IntValue): return reader.readShortArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeShort(value) + def writeArray(self, writer, font, tableDict, values): + writer.writeShortArray(values) class UShort(IntValue): staticSize = 2 @@ -266,6 +272,8 @@ class UShort(IntValue): return reader.readUShortArray(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeUShort(value) + def writeArray(self, writer, font, tableDict, values): + writer.writeUShortArray(values) class Int8(IntValue): staticSize = 1 @@ -275,6 +283,8 @@ class Int8(IntValue): return reader.readInt8Array(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeInt8(value) + def writeArray(self, writer, font, tableDict, values): + writer.writeInt8Array(values) class UInt8(IntValue): staticSize = 1 @@ -284,6 +294,8 @@ class UInt8(IntValue): return reader.readUInt8Array(count) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeUInt8(value) + def writeArray(self, writer, font, tableDict, values): + writer.writeUInt8Array(values) class UInt24(IntValue): staticSize = 3 From a8af308d70811c67b369f0941fe9955e5bfbaa9b Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 15:59:03 -0600 Subject: [PATCH 06/29] [otConverters] Use array-writers for ItemVariationStore payload --- Lib/fontTools/ttLib/tables/otConverters.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index ab2037f00..c5a44a5eb 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -1627,16 +1627,14 @@ class VarDataValue(BaseConverter): return values - def write(self, writer, font, tableDict, value, repeatIndex=None): + def write(self, writer, font, tableDict, values, repeatIndex=None): regionCount = tableDict["VarRegionCount"] shortCount = tableDict["NumShorts"] - for i in range(min(regionCount, shortCount)): - writer.writeShort(value[i]) - for i in range(min(regionCount, shortCount), regionCount): - writer.writeInt8(value[i]) - for i in range(regionCount, shortCount): - writer.writeInt8(0) + n1, n2 = min(regionCount, shortCount), max(regionCount, shortCount) + writer.writeShortArray(values[:n1]) + writer.writeInt8Array(values[n1:regionCount]) + writer.writeInt8Array([0] * (n2 - regionCount)) def xmlWrite(self, xmlWriter, font, value, name, attrs): xmlWriter.simpletag(name, attrs + [("value", value)]) From 81ca053039125d9ef2f72c5cb6c3a66c89f5ec32 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 16:26:49 -0600 Subject: [PATCH 07/29] [otBase] Add array reader/writer for UInt24 --- Lib/fontTools/ttLib/tables/otBase.py | 42 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index facef94fe..472648f84 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -184,7 +184,8 @@ class OTTableReader(object): value, = struct.unpack(">l", b'\0'+self.data[pos:newpos]) self.pos = newpos return value - + def readUInt24Array(self, count): + return [self.readUInt24() for _ in range(count)] def readTag(self): pos = self.pos @@ -433,11 +434,11 @@ class OTTableWriter(object): if sys.byteorder != "big": a.byteswap() self.items.append(a.tobytes()) - def writeUShort(self, value): - assert 0 <= value < 0x10000, value - self.items.append(struct.pack(">H", value)) - def writeUShortArray(self, values): - self.writeArray('H', values) + def writeInt8(self, value): + assert -128 <= value < 128, value + self.items.append(struct.pack(">b", value)) + def writeInt8Array(self, values): + self.writeArray('b', values) def writeShort(self, value): assert -32768 <= value < 32768, value @@ -445,28 +446,35 @@ class OTTableWriter(object): def writeShortArray(self, values): self.writeArray('h', values) + def writeLong(self, value): + self.items.append(struct.pack(">l", value)) + def writeLongArray(self, values): + self.writeArray('l', values) + def writeUInt8(self, value): assert 0 <= value < 256, value self.items.append(struct.pack(">B", value)) def writeUInt8Array(self, values): self.writeArray('B', values) - def writeInt8(self, value): - assert -128 <= value < 128, value - self.items.append(struct.pack(">b", value)) - def writeInt8Array(self, values): - self.writeArray('b', values) + def writeUShort(self, value): + assert 0 <= value < 0x10000, value + self.items.append(struct.pack(">H", value)) + def writeUShortArray(self, values): + self.writeArray('H', values) + + def writeULong(self, value): + self.items.append(struct.pack(">L", value)) + def writeULongArray(self, values): + self.writeArray('L', values) def writeUInt24(self, value): assert 0 <= value < 0x1000000, value b = struct.pack(">L", value) self.items.append(b[1:]) - - def writeLong(self, value): - self.items.append(struct.pack(">l", value)) - - def writeULong(self, value): - self.items.append(struct.pack(">L", value)) + def writeUInt24Array(self, values): + for value in values: + self.writeUInt24(value) def writeTag(self, tag): tag = Tag(tag).tobytes() From db6171df141b424e3f72748b4e83da577ad48157 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 16:28:28 -0600 Subject: [PATCH 08/29] [otBase] Actually call conv.writeArray() Huh. Somehow the writeArray() was never wired up. We lose the failing array index in the exception, but is fine to me. --- Lib/fontTools/ttLib/tables/otBase.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index 472648f84..e77b07a57 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -728,13 +728,12 @@ class BaseTable(object): # conv.repeat is a propagated count writer[conv.repeat].setValue(countValue) values = value - for i, value in enumerate(values): - try: - conv.write(writer, font, table, value, i) - except Exception as e: - name = value.__class__.__name__ if value is not None else conv.name - e.args = e.args + (name+'['+str(i)+']',) - raise + try: + conv.writeArray(writer, font, table, values) + except Exception as e: + name = value.__class__.__name__ if value is not None else conv.name + e.args = e.args + (name+'[]',) + raise elif conv.isCount: # Special-case Count values. # Assumption: a Count field will *always* precede From bd648ea14d6e747bfb15cb1d3041466ce151fcac Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 30 Apr 2021 16:29:18 -0600 Subject: [PATCH 09/29] [otConverters] Use array read/write in VarIdxMapValue --- Lib/fontTools/ttLib/tables/otConverters.py | 34 +++++++++------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index c5a44a5eb..855871051 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -1575,20 +1575,15 @@ class VarIdxMapValue(BaseConverter): outerShift = 16 - innerBits entrySize = 1 + ((fmt & 0x0030) >> 4) - read = { - 1: reader.readUInt8, - 2: reader.readUShort, - 3: reader.readUInt24, - 4: reader.readULong, + readArray = { + 1: reader.readUInt8Array, + 2: reader.readUShortArray, + 3: reader.readUInt24Array, + 4: reader.readULongArray, }[entrySize] - mapping = [] - for i in range(nItems): - raw = read() - idx = ((raw & outerMask) << outerShift) | (raw & innerMask) - mapping.append(idx) - - return mapping + return [(((raw & outerMask) << outerShift) | (raw & innerMask)) + for raw in readArray(nItems)] def write(self, writer, font, tableDict, value, repeatIndex=None): fmt = tableDict['EntryFormat'] @@ -1600,16 +1595,15 @@ class VarIdxMapValue(BaseConverter): outerShift = 16 - innerBits entrySize = 1 + ((fmt & 0x0030) >> 4) - write = { - 1: writer.writeUInt8, - 2: writer.writeUShort, - 3: writer.writeUInt24, - 4: writer.writeULong, + writeArray = { + 1: writer.writeUInt8Array, + 2: writer.writeUShortArray, + 3: writer.writeUInt24Array, + 4: writer.writeULongArray, }[entrySize] - for idx in mapping: - raw = ((idx & 0xFFFF0000) >> outerShift) | (idx & innerMask) - write(raw) + writeArray([(((idx & 0xFFFF0000) >> outerShift) | (idx & innerMask)) + for idx in mapping]) class VarDataValue(BaseConverter): From 0b20c196d4c6721e0e5502396a75e0c938ef25a3 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sat, 1 May 2021 12:41:45 -0600 Subject: [PATCH 10/29] [otConverters] Implement writeArray for GlyphID --- Lib/fontTools/ttLib/tables/otConverters.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index 855871051..448cfbfd4 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -338,6 +338,14 @@ class GlyphID(SimpleValue): return l def read(self, reader, font, tableDict): return font.getGlyphName(reader.readValue(self.typecode, self.staticSize)) + def writeArray(self, writer, font, tableDict, values): + glyphMap = font.getReverseGlyphMap() + try: + values = [glyphMap[glyphname] for glyphname in values] + except KeyError: + # Slower, but will not throw a KeyError on an out-of-range glyph name. + values = [font.getGlyphID(glyphname) for glyphname in values] + writer.writeArray(self.typecode, values) def write(self, writer, font, tableDict, value, repeatIndex=None): writer.writeValue(self.typecode, font.getGlyphID(value)) From 7d85b77996f2fc94aea4bcf5e24d9494ca8a81f8 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sat, 1 May 2021 12:44:15 -0600 Subject: [PATCH 11/29] [otConverters] Minor in VarStore padding --- Lib/fontTools/ttLib/tables/otConverters.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index 448cfbfd4..d135c9ae8 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -1625,7 +1625,8 @@ class VarDataValue(BaseConverter): n1, n2 = min(regionCount, shortCount), max(regionCount, shortCount) values.extend(reader.readShortArray(n1)) values.extend(reader.readInt8Array(n2 - n1)) - del values[regionCount:] + if n2 > regionCount: # Padding + del values[regionCount:] return values @@ -1636,7 +1637,8 @@ class VarDataValue(BaseConverter): n1, n2 = min(regionCount, shortCount), max(regionCount, shortCount) writer.writeShortArray(values[:n1]) writer.writeInt8Array(values[n1:regionCount]) - writer.writeInt8Array([0] * (n2 - regionCount)) + if n2 > regionCount: # Padding + writer.writeInt8Array([0] * (n2 - regionCount)) def xmlWrite(self, xmlWriter, font, value, name, attrs): xmlWriter.simpletag(name, attrs + [("value", value)]) From a2f34fdf822b9686775ca6edaddf9d021244cfca Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sat, 1 May 2021 12:45:03 -0600 Subject: [PATCH 12/29] [otConverters] Rename VarStore shortCount to wordCount in local variables --- Lib/fontTools/ttLib/tables/otConverters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index d135c9ae8..256895834 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -1620,9 +1620,9 @@ class VarDataValue(BaseConverter): values = [] regionCount = tableDict["VarRegionCount"] - shortCount = tableDict["NumShorts"] + wordCount = tableDict["NumShorts"] - n1, n2 = min(regionCount, shortCount), max(regionCount, shortCount) + n1, n2 = min(regionCount, wordCount), max(regionCount, wordCount) values.extend(reader.readShortArray(n1)) values.extend(reader.readInt8Array(n2 - n1)) if n2 > regionCount: # Padding @@ -1632,9 +1632,9 @@ class VarDataValue(BaseConverter): def write(self, writer, font, tableDict, values, repeatIndex=None): regionCount = tableDict["VarRegionCount"] - shortCount = tableDict["NumShorts"] + wordCount = tableDict["NumShorts"] - n1, n2 = min(regionCount, shortCount), max(regionCount, shortCount) + n1, n2 = min(regionCount, wordCount), max(regionCount, wordCount) writer.writeShortArray(values[:n1]) writer.writeInt8Array(values[n1:regionCount]) if n2 > regionCount: # Padding From 802e3636bc48abb792241bbda5d247184c8fb5fa Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sat, 1 May 2021 12:50:00 -0600 Subject: [PATCH 13/29] [otConverters] Support read/write of 32bit VarStore Part of https://github.com/fonttools/fonttools/issues/2279 --- Lib/fontTools/ttLib/tables/otConverters.py | 28 ++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index 256895834..af459856c 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -1622,9 +1622,18 @@ class VarDataValue(BaseConverter): regionCount = tableDict["VarRegionCount"] wordCount = tableDict["NumShorts"] + # https://github.com/fonttools/fonttools/issues/2279 + longWords = bool(wordCount & 0x8000) + wordCount = wordCount & 0x7FFF + + (readBigArray, readSmallArray) = { + False: (reader.readShortArray, reader.readInt8Array), + True: (reader.readLongArray, reader.readShortArray), + }[longWords] + n1, n2 = min(regionCount, wordCount), max(regionCount, wordCount) - values.extend(reader.readShortArray(n1)) - values.extend(reader.readInt8Array(n2 - n1)) + values.extend(readBigArray(n1)) + values.extend(readSmallArray(n2 - n1)) if n2 > regionCount: # Padding del values[regionCount:] @@ -1634,11 +1643,20 @@ class VarDataValue(BaseConverter): regionCount = tableDict["VarRegionCount"] wordCount = tableDict["NumShorts"] + # https://github.com/fonttools/fonttools/issues/2279 + longWords = bool(wordCount & 0x8000) + wordCount = wordCount & 0x7FFF + + (writeBigArray, writeSmallArray) = { + False: (writer.writeShortArray, writer.writeInt8Array), + True: (writer.writeLongArray, writer.writeShortArray), + }[longWords] + n1, n2 = min(regionCount, wordCount), max(regionCount, wordCount) - writer.writeShortArray(values[:n1]) - writer.writeInt8Array(values[n1:regionCount]) + writeBigArray(values[:n1]) + writeSmallArray(values[n1:regionCount]) if n2 > regionCount: # Padding - writer.writeInt8Array([0] * (n2 - regionCount)) + writer.writeSmallArray([0] * (n2 - regionCount)) def xmlWrite(self, xmlWriter, font, value, name, attrs): xmlWriter.simpletag(name, attrs + [("value", value)]) From e454e96238fd945e924c6aaa5d4b88707156f4de Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sat, 1 May 2021 14:27:11 -0600 Subject: [PATCH 14/29] [varLib.builder] Implement building 32bit VarStore The full optimizer in varLib.varStore still needs to be updated. But this pretty much enables building 32bit VarStores, even if they won't be fully optimal. Part of https://github.com/fonttools/fonttools/issues/2279 --- Lib/fontTools/varLib/builder.py | 49 +++++++++++++++++---------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/Lib/fontTools/varLib/builder.py b/Lib/fontTools/varLib/builder.py index 152336b06..45a9fb3d8 100644 --- a/Lib/fontTools/varLib/builder.py +++ b/Lib/fontTools/varLib/builder.py @@ -26,39 +26,40 @@ def buildVarRegionList(supports, axisTags): return self -def _reorderItem(lst, narrows, zeroes): - out = [] - count = len(lst) - for i in range(count): - if i not in narrows: - out.append(lst[i]) - for i in range(count): - if i in narrows and i not in zeroes: - out.append(lst[i]) - return out +def _reorderItem(lst, mapping): + return [lst[i] for i in mapping] def VarData_calculateNumShorts(self, optimize=False): count = self.VarRegionCount items = self.Item - narrows = set(range(count)) - zeroes = set(range(count)) + bit_lengths = [0] * count for item in items: - wides = [i for i in narrows if not (-128 <= item[i] <= 127)] - narrows.difference_update(wides) - nonzeroes = [i for i in zeroes if item[i]] - zeroes.difference_update(nonzeroes) - if not narrows and not zeroes: - break + bl = [(i + (i < -1)).bit_length() for i in item] + bit_lengths = [max(*pair) for pair in zip(bl, bit_lengths)] + byte_lengths = [((b + 8) // 8) if b else 0 for b in bit_lengths] + + # https://github.com/fonttools/fonttools/issues/2279 + longWords = any(b > 2 for b in byte_lengths) + if optimize: - # Reorder columns such that all SHORT columns come before UINT8 - self.VarRegionIndex = _reorderItem(self.VarRegionIndex, narrows, zeroes) + # Reorder columns such that wider columns come before narrower columns + mapping = [] + mapping.extend(i for i,b in enumerate(byte_lengths) if b > 2) + mapping.extend(i for i,b in enumerate(byte_lengths) if b == 2) + mapping.extend(i for i,b in enumerate(byte_lengths) if b == 1) + + byte_lengths = _reorderItem(byte_lengths, mapping) + self.VarRegionIndex = _reorderItem(self.VarRegionIndex, mapping) self.VarRegionCount = len(self.VarRegionIndex) for i in range(len(items)): - items[i] = _reorderItem(items[i], narrows, zeroes) - self.NumShorts = count - len(narrows) + items[i] = _reorderItem(items[i], mapping) + + if longWords: + self.NumShorts = max((i for i,b in enumerate(byte_lengths) if b > 2), default=-1) + 1 + self.NumShorts |= 0x8000 else: - wides = set(range(count)) - narrows - self.NumShorts = 1+max(wides) if wides else 0 + self.NumShorts = max((i for i,b in enumerate(byte_lengths) if b > 1), default=-1) + 1 + self.VarRegionCount = len(self.VarRegionIndex) return self From 9350166792f49ba4acc674f73aa00f5513109add Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sat, 1 May 2021 15:04:51 -0600 Subject: [PATCH 15/29] [varLib.varStore] Remove use of array.array --- Lib/fontTools/varLib/varStore.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/fontTools/varLib/varStore.py b/Lib/fontTools/varLib/varStore.py index 8a382df01..27b6eadee 100644 --- a/Lib/fontTools/varLib/varStore.py +++ b/Lib/fontTools/varLib/varStore.py @@ -5,7 +5,6 @@ from fontTools.varLib.builder import (buildVarRegionList, buildVarStore, buildVarRegion, buildVarData) from functools import partial from collections import defaultdict -from array import array def _getLocationKey(loc): @@ -423,7 +422,7 @@ def VarStore_optimize(self): # Check that no two VarRegions are the same; if they are, fold them. n = len(self.VarRegionList.Region) # Number of columns - zeroes = array('h', [0]*n) + zeroes = [0] * n front_mapping = {} # Map from old VarIdxes to full row tuples @@ -435,7 +434,7 @@ def VarStore_optimize(self): for minor,item in enumerate(data.Item): - row = array('h', zeroes) + row = list(zeroes) for regionIdx,v in zip(regionIndices, item): row[regionIdx] += v row = tuple(row) From 0549b27afbeea84646960934478d84fad815d7b0 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Sat, 1 May 2021 15:12:08 -0600 Subject: [PATCH 16/29] [varLib.varStore] Implement 32bit VarStore optimization This concludes https://github.com/fonttools/fonttools/issues/2279 Part of https://github.com/fonttools/fonttools/pull/2285 --- Lib/fontTools/varLib/varStore.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/Lib/fontTools/varLib/varStore.py b/Lib/fontTools/varLib/varStore.py index 27b6eadee..47f9e8df3 100644 --- a/Lib/fontTools/varLib/varStore.py +++ b/Lib/fontTools/varLib/varStore.py @@ -374,12 +374,11 @@ class _Encoding(object): as a VarData.""" c = 6 while chars: - if chars & 3: + if chars & 15: c += 2 - chars >>= 2 + chars >>= 4 return c - def _find_yourself_best_new_encoding(self, done_by_width): self.best_new_encoding = None for new_width in range(self.width+1, self.width+self.room+1): @@ -404,6 +403,8 @@ class _EncodingDict(dict): @staticmethod def _row_characteristics(row): """Returns encoding characteristics for a row.""" + longWords = False + chars = 0 i = 1 for v in row: @@ -411,7 +412,22 @@ class _EncodingDict(dict): chars += i if not (-128 <= v <= 127): chars += i * 2 - i <<= 2 + if not (-32768 <= v <= 32767): + longWords = True + break + i <<= 4 + + if longWords: + # Redo; only allow 2byte/4byte encoding + chars = 0 + i = 1 + for v in row: + if v: + chars += i * 3 # 3 = 2 | 1 + if not (-32768 <= v <= 32767): + chars += i * 12 # 12 = 8 | 4 + i <<= 4 + return chars From eba058d43921c465e5a72cc8bd3d254b572dd342 Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 14:41:01 +0200 Subject: [PATCH 17/29] add 32-bit tests to test_buildVarData_no_optimize --- Tests/varLib/builder_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tests/varLib/builder_test.py b/Tests/varLib/builder_test.py index 80607eaad..eecc5040c 100644 --- a/Tests/varLib/builder_test.py +++ b/Tests/varLib/builder_test.py @@ -8,12 +8,20 @@ import pytest ([0], [[128]], 1), ([0, 1, 2], [[128, 1, 2], [3, -129, 5], [6, 7, 8]], 2), ([0, 1, 2], [[0, 128, 2], [3, 4, 5], [6, 7, -129]], 3), + ([0], [[32768]], 0x8001), + ([0, 1, 2], [[32768, 1, 2], [3, -129, 5], [6, 7, 8]], 0x8001), + ([0, 1, 2], [[32768, 1, 2], [3, -32769, 5], [6, 7, 8]], 0x8002), + ([0, 1, 2], [[0, 32768, 2], [3, 4, 5], [6, 7, -32769]], 0x8003), ], ids=[ "0_regions_0_deltas", "1_region_1_uint8", "1_region_1_short", "3_regions_2_shorts_ordered", "3_regions_2_shorts_unordered", + "1_region_1_long", + "3_regions_1_long_ordered", + "3_regions_2_longs_ordered", + "3_regions_2_longs_unordered", ]) def test_buildVarData_no_optimize(region_indices, items, expected_num_shorts): data = buildVarData(region_indices, items, optimize=False) From a7fd20295274b076a0185c7bfd2c7daa2a85a26b Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 14:49:12 +0200 Subject: [PATCH 18/29] add 32-bit tests to test_buildVarData_optimize --- Tests/varLib/builder_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Tests/varLib/builder_test.py b/Tests/varLib/builder_test.py index eecc5040c..6cad103ab 100644 --- a/Tests/varLib/builder_test.py +++ b/Tests/varLib/builder_test.py @@ -49,6 +49,16 @@ def test_buildVarData_no_optimize(region_indices, items, expected_num_shorts): [0, 1, 2], [[0, 1, 128], [3, -129, 5], [256, 7, 8]]), ([0, 1, 2], [[0, 128, 2], [0, 4, 5], [0, 7, 8]], 1, [1, 2], [[128, 2], [4, 5], [7, 8]]), + ([0, 1, 2], [[0, 32768, 2], [3, 4, 5], [6, 7, 8]], 0x8001, + [1, 0, 2], [[32768, 0, 2], [4, 3, 5], [7, 6, 8]]), + ([0, 1, 2], [[0, 1, 32768], [3, 4, 5], [6, -32769, 8]], 0x8002, + [1, 2, 0], [[1, 32768, 0], [4, 5, 3], [-32769, 8, 6]]), + ([0, 1, 2], [[32768, 1, -32769], [3, 4, 5], [6, 7, 8]], 0x8002, + [0, 2, 1], [[32768, -32769, 1], [3, 5, 4], [6, 8, 7]]), + ([0, 1, 2], [[0, 1, 32768], [3, -32769, 5], [65536, 7, 8]], 0x8003, + [0, 1, 2], [[0, 1, 32768], [3, -32769, 5], [65536, 7, 8]]), + ([0, 1, 2], [[0, 32768, 2], [0, 4, 5], [0, 7, 8]], 0x8001, + [1, 2], [[32768, 2], [4, 5], [7, 8]]), ], ids=[ "0/3_shorts_no_reorder", "1/3_shorts_reorder", @@ -56,6 +66,11 @@ def test_buildVarData_no_optimize(region_indices, items, expected_num_shorts): "2/3_shorts_same_row_reorder", "3/3_shorts_no_reorder", "1/3_shorts_1/3_zeroes", + "1/3_longs_reorder", + "2/3_longs_reorder", + "2/3_longs_same_row_reorder", + "3/3_longs_no_reorder", + "1/3_longs_1/3_zeroes", ]) def test_buildVarData_optimize( region_indices, items, expected_num_shorts, expected_regions, From 7de2f347c560923b7e702475ad1fbfd3fc515552 Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 16:12:24 +0200 Subject: [PATCH 19/29] add tests for OnlineVarStoreBuilder/VarStoreInstancer --- Tests/varLib/varStore_test.py | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 Tests/varLib/varStore_test.py diff --git a/Tests/varLib/varStore_test.py b/Tests/varLib/varStore_test.py new file mode 100644 index 000000000..631d70100 --- /dev/null +++ b/Tests/varLib/varStore_test.py @@ -0,0 +1,60 @@ +import pytest +from fontTools.varLib.models import VariationModel +from fontTools.varLib.varStore import OnlineVarStoreBuilder, VarStoreInstancer +from fontTools.ttLib.tables._f_v_a_r import Axis + + +@pytest.mark.parametrize( + "locations, masterValues", + [ + ( + [{}, {"a": 1}], + [ + [10, 20], + [100, 2000], + [100, 22000], + ], + ), + ( + [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}], + [ + [10, 20, 40, 60], + [100, 2000, 400, 6000], + [100, 22000, 4000, 30000], + ], + ), + ], +) +def test_onlineVarStoreBuilder(locations, masterValues): + axisTags = sorted({k for loc in locations for k in loc}) + model = VariationModel(locations) + builder = OnlineVarStoreBuilder(axisTags) + builder.setModel(model) + expectedDeltasAndVarIdxs = [] + for masters in masterValues: + base, *deltas = model.getDeltas(masters) + varIdx = builder.storeDeltas(deltas) + expectedDeltasAndVarIdxs.append((deltas, varIdx)) + + varStore = builder.finish() + varData = varStore.VarData + + for deltas, varIdx in expectedDeltasAndVarIdxs: + major, minor = varIdx >> 16, varIdx & 0xFFFF + storedDeltas = varData[major].Item[minor] + assert deltas == storedDeltas + + fvarAxes = [buildAxis(axisTag) for axisTag in axisTags] + instancer = VarStoreInstancer(varStore, fvarAxes) + for masters, (deltas, varIdx) in zip(masterValues, expectedDeltasAndVarIdxs): + base, *rest = masters + for expectedValue, loc in zip(masters, locations): + instancer.setLocation(loc) + value = base + instancer[varIdx] + assert expectedValue == value + + +def buildAxis(axisTag): + axis = Axis() + axis.axisTag = axisTag + return axis From 22dda546160c20b913d871628edded68f0b2d6fd Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 16:19:48 +0200 Subject: [PATCH 20/29] use optimizer --- Tests/varLib/varStore_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Tests/varLib/varStore_test.py b/Tests/varLib/varStore_test.py index 631d70100..88c6a64f7 100644 --- a/Tests/varLib/varStore_test.py +++ b/Tests/varLib/varStore_test.py @@ -37,11 +37,14 @@ def test_onlineVarStoreBuilder(locations, masterValues): expectedDeltasAndVarIdxs.append((deltas, varIdx)) varStore = builder.finish() - varData = varStore.VarData + mapping = varStore.optimize() + expectedDeltasAndVarIdxs = [ + (deltas, mapping[varIdx]) for deltas, varIdx in expectedDeltasAndVarIdxs + ] for deltas, varIdx in expectedDeltasAndVarIdxs: major, minor = varIdx >> 16, varIdx & 0xFFFF - storedDeltas = varData[major].Item[minor] + storedDeltas = varStore.VarData[major].Item[minor] assert deltas == storedDeltas fvarAxes = [buildAxis(axisTag) for axisTag in axisTags] From 267ab2babac37fbc423bef9555a4b16c593fa5f1 Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 16:39:20 +0200 Subject: [PATCH 21/29] add 32-bit master value tests --- Tests/varLib/varStore_test.py | 36 +++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/Tests/varLib/varStore_test.py b/Tests/varLib/varStore_test.py index 88c6a64f7..8b593314b 100644 --- a/Tests/varLib/varStore_test.py +++ b/Tests/varLib/varStore_test.py @@ -20,7 +20,23 @@ from fontTools.ttLib.tables._f_v_a_r import Axis [ [10, 20, 40, 60], [100, 2000, 400, 6000], - [100, 22000, 4000, 30000], + [177100, 22000, 4000, 30000], + ], + ), + ( + [{}, {"a": 1}], + [ + [10, 20], + [42000, 100], + [100, 52000], + ], + ), + ( + [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}], + [ + [10, 20, 40, 60], + [40000, 42000, 400, 6000], + [100, 22000, 4000, 173000], ], ), ], @@ -30,26 +46,18 @@ def test_onlineVarStoreBuilder(locations, masterValues): model = VariationModel(locations) builder = OnlineVarStoreBuilder(axisTags) builder.setModel(model) - expectedDeltasAndVarIdxs = [] + varIdxs = [] for masters in masterValues: - base, *deltas = model.getDeltas(masters) - varIdx = builder.storeDeltas(deltas) - expectedDeltasAndVarIdxs.append((deltas, varIdx)) + _, varIdx = builder.storeMasters(masters) + varIdxs.append(varIdx) varStore = builder.finish() mapping = varStore.optimize() - expectedDeltasAndVarIdxs = [ - (deltas, mapping[varIdx]) for deltas, varIdx in expectedDeltasAndVarIdxs - ] - - for deltas, varIdx in expectedDeltasAndVarIdxs: - major, minor = varIdx >> 16, varIdx & 0xFFFF - storedDeltas = varStore.VarData[major].Item[minor] - assert deltas == storedDeltas + varIdxs = [mapping[varIdx] for varIdx in varIdxs] fvarAxes = [buildAxis(axisTag) for axisTag in axisTags] instancer = VarStoreInstancer(varStore, fvarAxes) - for masters, (deltas, varIdx) in zip(masterValues, expectedDeltasAndVarIdxs): + for masters, varIdx in zip(masterValues, varIdxs): base, *rest = masters for expectedValue, loc in zip(masters, locations): instancer.setLocation(loc) From ad438931e476be11efebe5da122b5d2a33aeb5dd Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 17:08:27 +0200 Subject: [PATCH 22/29] adding compile/decompile step; this currently fails for the 32-bit tests (which are therefore commented out) --- Tests/varLib/varStore_test.py | 48 ++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/Tests/varLib/varStore_test.py b/Tests/varLib/varStore_test.py index 8b593314b..948a6eaac 100644 --- a/Tests/varLib/varStore_test.py +++ b/Tests/varLib/varStore_test.py @@ -1,7 +1,10 @@ import pytest from fontTools.varLib.models import VariationModel from fontTools.varLib.varStore import OnlineVarStoreBuilder, VarStoreInstancer +from fontTools.ttLib import TTFont, newTable from fontTools.ttLib.tables._f_v_a_r import Axis +from fontTools.ttLib.tables.otBase import OTTableReader, OTTableWriter +from fontTools.ttLib.tables.otTables import VarStore @pytest.mark.parametrize( @@ -20,25 +23,25 @@ from fontTools.ttLib.tables._f_v_a_r import Axis [ [10, 20, 40, 60], [100, 2000, 400, 6000], - [177100, 22000, 4000, 30000], - ], - ), - ( - [{}, {"a": 1}], - [ - [10, 20], - [42000, 100], - [100, 52000], - ], - ), - ( - [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}], - [ - [10, 20, 40, 60], - [40000, 42000, 400, 6000], - [100, 22000, 4000, 173000], + [7100, 22000, 4000, 30000], ], ), + # ( + # [{}, {"a": 1}], + # [ + # [10, 20], + # [42000, 100], + # [100, 52000], + # ], + # ), + # ( + # [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}], + # [ + # [10, 20, 40, 60], + # [40000, 42000, 400, 6000], + # [100, 22000, 4000, 173000], + # ], + # ), ], ) def test_onlineVarStoreBuilder(locations, masterValues): @@ -55,7 +58,18 @@ def test_onlineVarStoreBuilder(locations, masterValues): mapping = varStore.optimize() varIdxs = [mapping[varIdx] for varIdx in varIdxs] + font = TTFont() fvarAxes = [buildAxis(axisTag) for axisTag in axisTags] + font["fvar"] = newTable("fvar") + font["fvar"].axes = fvarAxes + + writer = OTTableWriter() + varStore.compile(writer, font) + data = writer.getAllData() + reader = OTTableReader(data) + varStore = VarStore() + varStore.decompile(reader, font) + instancer = VarStoreInstancer(varStore, fvarAxes) for masters, varIdx in zip(masterValues, varIdxs): base, *rest = masters From d3fd46e3f33810e9a15773689f40d0c04e5ce0ad Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 18:21:37 +0200 Subject: [PATCH 23/29] uncomment failing tests --- Tests/varLib/varStore_test.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Tests/varLib/varStore_test.py b/Tests/varLib/varStore_test.py index 948a6eaac..c6289a095 100644 --- a/Tests/varLib/varStore_test.py +++ b/Tests/varLib/varStore_test.py @@ -26,22 +26,22 @@ from fontTools.ttLib.tables.otTables import VarStore [7100, 22000, 4000, 30000], ], ), - # ( - # [{}, {"a": 1}], - # [ - # [10, 20], - # [42000, 100], - # [100, 52000], - # ], - # ), - # ( - # [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}], - # [ - # [10, 20, 40, 60], - # [40000, 42000, 400, 6000], - # [100, 22000, 4000, 173000], - # ], - # ), + ( + [{}, {"a": 1}], + [ + [10, 20], + [42000, 100], + [100, 52000], + ], + ), + ( + [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}], + [ + [10, 20, 40, 60], + [40000, 42000, 400, 6000], + [100, 22000, 4000, 173000], + ], + ), ], ) def test_onlineVarStoreBuilder(locations, masterValues): From 747f9f49b70eb12cba46298d056aaf60d9df9c0f Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 18:54:26 +0200 Subject: [PATCH 24/29] fix struct vs array mismatch: for array 'l' is 8 bytes, not 4. I'm not 100% sure this is correct for all platforms. --- Lib/fontTools/ttLib/tables/otBase.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index e77b07a57..29b8998af 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -161,7 +161,7 @@ class OTTableReader(object): def readLong(self): return self.readValue("l", staticSize=4) def readLongArray(self, count): - return self.readArray("l", staticSize=4, count=count) + return self.readArray("i", staticSize=4, count=count) def readUInt8(self): return self.readValue("B", staticSize=1) @@ -176,7 +176,7 @@ class OTTableReader(object): def readULong(self): return self.readValue("L", staticSize=4) def readULongArray(self, count): - return self.readArray("L", staticSize=4, count=count) + return self.readArray("I", staticSize=4, count=count) def readUInt24(self): pos = self.pos @@ -449,7 +449,7 @@ class OTTableWriter(object): def writeLong(self, value): self.items.append(struct.pack(">l", value)) def writeLongArray(self, values): - self.writeArray('l', values) + self.writeArray('i', values) def writeUInt8(self, value): assert 0 <= value < 256, value @@ -466,7 +466,7 @@ class OTTableWriter(object): def writeULong(self, value): self.items.append(struct.pack(">L", value)) def writeULongArray(self, values): - self.writeArray('L', values) + self.writeArray('I', values) def writeUInt24(self, value): assert 0 <= value < 0x1000000, value From 26363258444ef9cb425b0965686cbd014d9af168 Mon Sep 17 00:00:00 2001 From: justvanrossum Date: Tue, 4 May 2021 19:29:39 +0200 Subject: [PATCH 25/29] font placeholder doesn't need an fvar table --- Tests/varLib/varStore_test.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Tests/varLib/varStore_test.py b/Tests/varLib/varStore_test.py index c6289a095..a1bdd1237 100644 --- a/Tests/varLib/varStore_test.py +++ b/Tests/varLib/varStore_test.py @@ -58,18 +58,15 @@ def test_onlineVarStoreBuilder(locations, masterValues): mapping = varStore.optimize() varIdxs = [mapping[varIdx] for varIdx in varIdxs] - font = TTFont() - fvarAxes = [buildAxis(axisTag) for axisTag in axisTags] - font["fvar"] = newTable("fvar") - font["fvar"].axes = fvarAxes - + dummyFont = TTFont() writer = OTTableWriter() - varStore.compile(writer, font) + varStore.compile(writer, dummyFont) data = writer.getAllData() reader = OTTableReader(data) varStore = VarStore() - varStore.decompile(reader, font) + varStore.decompile(reader, dummyFont) + fvarAxes = [buildAxis(axisTag) for axisTag in axisTags] instancer = VarStoreInstancer(varStore, fvarAxes) for masters, varIdx in zip(masterValues, varIdxs): base, *rest = masters From b8963256fd1583efd72d3fd005446ec4925816bd Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 7 May 2021 15:31:53 -0600 Subject: [PATCH 26/29] [otBase/otConverters] Add back array index to exception in writeArray() https://github.com/fonttools/fonttools/pull/2285/commits/db6171df141b424e3f72748b4e83da577ad48157#r628543432 --- Lib/fontTools/ttLib/tables/otBase.py | 6 ++---- Lib/fontTools/ttLib/tables/otConverters.py | 8 ++++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index 29b8998af..72482bca0 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -727,12 +727,10 @@ class BaseTable(object): else: # conv.repeat is a propagated count writer[conv.repeat].setValue(countValue) - values = value try: - conv.writeArray(writer, font, table, values) + conv.writeArray(writer, font, table, value) except Exception as e: - name = value.__class__.__name__ if value is not None else conv.name - e.args = e.args + (name+'[]',) + e.args = e.args + (conv.name+'[]',) raise elif conv.isCount: # Special-case Count values. diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index af459856c..d5bfc5aa5 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -192,8 +192,12 @@ class BaseConverter(object): raise NotImplementedError(self) def writeArray(self, writer, font, tableDict, values): - for i, value in enumerate(values): - self.write(writer, font, tableDict, value, i) + try: + for i, value in enumerate(values): + self.write(writer, font, tableDict, value, i) + except Exception as e: + e.args = e.args + (i,) + raise def write(self, writer, font, tableDict, value, repeatIndex=None): """Write a value to the writer.""" From 98e2bf5526a49a89b498801c357422340d7bb5df Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 7 May 2021 15:41:11 -0600 Subject: [PATCH 27/29] [varLib.builder] Document bit-tweedling https://github.com/fonttools/fonttools/pull/2285#discussion_r628401241 --- Lib/fontTools/ttLib/tables/otConverters.py | 8 ++++---- Lib/fontTools/varLib/builder.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otConverters.py b/Lib/fontTools/ttLib/tables/otConverters.py index d5bfc5aa5..1f1a3194b 100644 --- a/Lib/fontTools/ttLib/tables/otConverters.py +++ b/Lib/fontTools/ttLib/tables/otConverters.py @@ -1630,10 +1630,10 @@ class VarDataValue(BaseConverter): longWords = bool(wordCount & 0x8000) wordCount = wordCount & 0x7FFF - (readBigArray, readSmallArray) = { - False: (reader.readShortArray, reader.readInt8Array), - True: (reader.readLongArray, reader.readShortArray), - }[longWords] + if longWords: + readBigArray, readSmallArray = reader.readLongArray, reader.readShortArray + else: + readBigArray, readSmallArray = reader.readShortArray, reader.readInt8Array n1, n2 = min(regionCount, wordCount), max(regionCount, wordCount) values.extend(readBigArray(n1)) diff --git a/Lib/fontTools/varLib/builder.py b/Lib/fontTools/varLib/builder.py index 45a9fb3d8..c024c613d 100644 --- a/Lib/fontTools/varLib/builder.py +++ b/Lib/fontTools/varLib/builder.py @@ -34,9 +34,23 @@ def VarData_calculateNumShorts(self, optimize=False): items = self.Item bit_lengths = [0] * count for item in items: + # The "+ (i < -1)" magic is to handle two's-compliment. + # That is, we want to get back 7 for -128, whereas + # bit_length() returns 8. Similarly for -65536. + # The reason "i < -1" is used instead of "i < 0" is that + # the latter would make it return 0 for "-1" instead of 1. bl = [(i + (i < -1)).bit_length() for i in item] bit_lengths = [max(*pair) for pair in zip(bl, bit_lengths)] - byte_lengths = [((b + 8) // 8) if b else 0 for b in bit_lengths] + # The addition of 8, instead of seven, is to account for the sign bit. + # This "((b + 8) >> 3) if b else 0" when combined with the above + # "(i + (i < -1)).bit_length()" is a faster way to compute byte-lengths + # conforming to: + # + # byte_length = (0 if i == 0 else + # 1 if -128 <= i < 128 else + # 2 if -65536 <= i < 65536 else + # ...) + byte_lengths = [((b + 8) >> 3) if b else 0 for b in bit_lengths] # https://github.com/fonttools/fonttools/issues/2279 longWords = any(b > 2 for b in byte_lengths) From 03e97edf145b8454b2650ad485a349af79a13423 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 7 May 2021 15:45:55 -0600 Subject: [PATCH 28/29] [varLib.varStore] Use binary notation for bit constants https://github.com/fonttools/fonttools/pull/2285#discussion_r625708297 --- Lib/fontTools/varLib/varStore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/fontTools/varLib/varStore.py b/Lib/fontTools/varLib/varStore.py index 47f9e8df3..bcf81b39d 100644 --- a/Lib/fontTools/varLib/varStore.py +++ b/Lib/fontTools/varLib/varStore.py @@ -374,7 +374,7 @@ class _Encoding(object): as a VarData.""" c = 6 while chars: - if chars & 15: + if chars & 0b1111: c += 2 chars >>= 4 return c @@ -411,7 +411,7 @@ class _EncodingDict(dict): if v: chars += i if not (-128 <= v <= 127): - chars += i * 2 + chars += i * 0b0010 if not (-32768 <= v <= 32767): longWords = True break @@ -423,9 +423,9 @@ class _EncodingDict(dict): i = 1 for v in row: if v: - chars += i * 3 # 3 = 2 | 1 + chars += i * 0b0011 if not (-32768 <= v <= 32767): - chars += i * 12 # 12 = 8 | 4 + chars += i * 0b1100 i <<= 4 return chars From f21091b6ff160586d64342ffbe4efedd580cde79 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 7 May 2021 15:52:35 -0600 Subject: [PATCH 29/29] [otBase] Assert array.array('i').itemsize --- Lib/fontTools/ttLib/tables/otBase.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/fontTools/ttLib/tables/otBase.py b/Lib/fontTools/ttLib/tables/otBase.py index 72482bca0..06d5cde54 100644 --- a/Lib/fontTools/ttLib/tables/otBase.py +++ b/Lib/fontTools/ttLib/tables/otBase.py @@ -106,6 +106,10 @@ class BaseTTXConverter(DefaultTable): self.table.populateDefaults() +# https://github.com/fonttools/fonttools/pull/2285#issuecomment-834652928 +assert len(struct.pack('i', 0)) == 4 +assert array.array('i').itemsize == 4, "Oops, file a bug against fonttools." + class OTTableReader(object): """Helper class to retrieve data from an OpenType table.""" @@ -159,7 +163,7 @@ class OTTableReader(object): return self.readArray("h", staticSize=2, count=count) def readLong(self): - return self.readValue("l", staticSize=4) + return self.readValue("i", staticSize=4) def readLongArray(self, count): return self.readArray("i", staticSize=4, count=count) @@ -174,7 +178,7 @@ class OTTableReader(object): return self.readArray("H", staticSize=2, count=count) def readULong(self): - return self.readValue("L", staticSize=4) + return self.readValue("I", staticSize=4) def readULongArray(self, count): return self.readArray("I", staticSize=4, count=count) @@ -447,7 +451,7 @@ class OTTableWriter(object): self.writeArray('h', values) def writeLong(self, value): - self.items.append(struct.pack(">l", value)) + self.items.append(struct.pack(">i", value)) def writeLongArray(self, values): self.writeArray('i', values) @@ -464,7 +468,7 @@ class OTTableWriter(object): self.writeArray('H', values) def writeULong(self, value): - self.items.append(struct.pack(">L", value)) + self.items.append(struct.pack(">I", value)) def writeULongArray(self, values): self.writeArray('I', values) @@ -561,11 +565,11 @@ def packUShort(value): def packULong(value): assert 0 <= value < 0x100000000, value - return struct.pack(">L", value) + return struct.pack(">I", value) def packUInt24(value): assert 0 <= value < 0x1000000, value - return struct.pack(">L", value)[1:] + return struct.pack(">I", value)[1:] class BaseTable(object):