[varStore] Handle >65535 items per encoding

By creating a new major for each 65535.

Lib/fontTools/varLib/varStore.py

@@ -619,14 +619,21 @@ def VarStore_optimize(self, use_NO_VARIATION_INDEX=True, quantization=1):
     back_mapping = {}  # Mapping from full rows to new VarIdxes
     encodings.sort(key=_Encoding.width_sort_key)
     self.VarData = []
-    for major, encoding in enumerate(encodings):
-        data = ot.VarData()
-        self.VarData.append(data)
-        data.VarRegionIndex = range(n)
-        data.VarRegionCount = len(data.VarRegionIndex)
-        data.Item = sorted(encoding.items)
-        for minor, item in enumerate(data.Item):
-            back_mapping[item] = (major << 16) + minor
+    for encoding in encodings:
+        items = sorted(encoding.items)
+
+        while items:
+            major = len(self.VarData)
+            data = ot.VarData()
+            self.VarData.append(data)
+            data.VarRegionIndex = range(n)
+            data.VarRegionCount = len(data.VarRegionIndex)
+
+            # Each major can only encode up to 0xFFFF entries.
+            data.Item, items = items[:0xFFFF], items[0xFFFF:]
+
+            for minor, item in enumerate(data.Item):
+                back_mapping[item] = (major << 16) + minor
 
     # Compile final mapping.
     varidx_map = {NO_VARIATION_INDEX: NO_VARIATION_INDEX}

Tests/varLib/varStore_test.py

@@ -256,3 +256,30 @@ def test_quantize(quantization, expectedBytes):
     data = writer.getAllData()
 
     assert len(data) == expectedBytes, xml
+
+
+def test_optimize_overflow():
+    numRegions = 1
+    locations = [{"wght": 0}, {"wght": 0.5}]
+    axisTags = ["wght"]
+
+    model = VariationModel(locations)
+    builder = OnlineVarStoreBuilder(axisTags)
+    builder.setModel(model)
+
+    for data in range(0, 0xFFFF * 2):
+        data = [0, data]
+        builder.storeMasters(data)
+
+    varStore = builder.finish()
+    varStore.optimize()
+
+    for s in varStore.VarData:
+        print(len(s.Item))
+
+    # 5 data-sets:
+    # - 0..127: 1-byte dataset
+    # - 128..32767: 2-byte dataset
+    # - 32768..32768+65535-1: 4-byte dataset
+    # - 32768+65535..65535+65535-1: 4-byte dataset
+    assert len(varStore.VarData) == 4