From c7ce5b0f3c84870aec4257b8501bd12a952e3136 Mon Sep 17 00:00:00 2001
From: Behdad Esfahbod <behdad@behdad.org>
Date: Mon, 23 Oct 2023 13:41:16 -0600
Subject: [PATCH] [varStore] Handle >65535 items per encoding

By creating a new major for each 65535.
---
 Lib/fontTools/varLib/varStore.py | 23 +++++++++++++++--------
 Tests/varLib/varStore_test.py    | 27 +++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/Lib/fontTools/varLib/varStore.py b/Lib/fontTools/varLib/varStore.py
index 74828e407..1fe8e61b9 100644
--- a/Lib/fontTools/varLib/varStore.py
+++ b/Lib/fontTools/varLib/varStore.py
@@ -619,14 +619,21 @@ def VarStore_optimize(self, use_NO_VARIATION_INDEX=True, quantization=1):
     back_mapping = {}  # Mapping from full rows to new VarIdxes
     encodings.sort(key=_Encoding.width_sort_key)
     self.VarData = []
-    for major, encoding in enumerate(encodings):
-        data = ot.VarData()
-        self.VarData.append(data)
-        data.VarRegionIndex = range(n)
-        data.VarRegionCount = len(data.VarRegionIndex)
-        data.Item = sorted(encoding.items)
-        for minor, item in enumerate(data.Item):
-            back_mapping[item] = (major << 16) + minor
+    for encoding in encodings:
+        items = sorted(encoding.items)
+
+        while items:
+            major = len(self.VarData)
+            data = ot.VarData()
+            self.VarData.append(data)
+            data.VarRegionIndex = range(n)
+            data.VarRegionCount = len(data.VarRegionIndex)
+
+            # Each major can only encode up to 0xFFFF entries.
+            data.Item, items = items[:0xFFFF], items[0xFFFF:]
+
+            for minor, item in enumerate(data.Item):
+                back_mapping[item] = (major << 16) + minor
 
     # Compile final mapping.
     varidx_map = {NO_VARIATION_INDEX: NO_VARIATION_INDEX}
diff --git a/Tests/varLib/varStore_test.py b/Tests/varLib/varStore_test.py
index 75b6387d1..f248f5df1 100644
--- a/Tests/varLib/varStore_test.py
+++ b/Tests/varLib/varStore_test.py
@@ -256,3 +256,30 @@ def test_quantize(quantization, expectedBytes):
     data = writer.getAllData()
 
     assert len(data) == expectedBytes, xml
+
+
+def test_optimize_overflow():
+    numRegions = 1
+    locations = [{"wght": 0}, {"wght": 0.5}]
+    axisTags = ["wght"]
+
+    model = VariationModel(locations)
+    builder = OnlineVarStoreBuilder(axisTags)
+    builder.setModel(model)
+
+    for data in range(0, 0xFFFF * 2):
+        data = [0, data]
+        builder.storeMasters(data)
+
+    varStore = builder.finish()
+    varStore.optimize()
+
+    for s in varStore.VarData:
+        print(len(s.Item))
+
+    # 5 data-sets:
+    # - 0..127: 1-byte dataset
+    # - 128..32767: 2-byte dataset
+    # - 32768..32768+65535-1: 4-byte dataset
+    # - 32768+65535..65535+65535-1: 4-byte dataset
+    assert len(varStore.VarData) == 4