From e5029801d2c1739b8a51de926b3cb636edf2926e Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Tue, 5 Jul 2022 00:11:12 +0100 Subject: [PATCH] support merging COLR masters with 'sparse' glyphsets or different layer count There is no longer a requirement that all the masters have exactly the same base color glyphs as the default masters. Similarly, it's no longer required that all masters' LayerLists have the same total count of layers. It is sufficient that, for a base color glyph in the default master, a non-default master may (or may not) contain one with the same name and same effective number of layers (which in turn can be laid out differently in the respective LayerLists). This provides greater flexibility when working with variable font project with sparse glyph sets. --- Lib/fontTools/varLib/__init__.py | 7 +- Lib/fontTools/varLib/merger.py | 130 +++++++- Tests/varLib/merger_test.py | 551 ++++++++++++++++++++++++++++++- 3 files changed, 680 insertions(+), 8 deletions(-) diff --git a/Lib/fontTools/varLib/__init__.py b/Lib/fontTools/varLib/__init__.py index a98a45b8d..367d6b1bc 100644 --- a/Lib/fontTools/varLib/__init__.py +++ b/Lib/fontTools/varLib/__init__.py @@ -714,7 +714,7 @@ def _add_CFF2(varFont, model, master_fonts): def _add_COLR(font, model, master_fonts, axisTags, colr_layer_reuse=True): - merger = COLRVariationMerger(model, axisTags, font) + merger = COLRVariationMerger(model, axisTags, font, allowLayerReuse=colr_layer_reuse) merger.mergeTables(font, master_fonts) store = merger.store_builder.finish() @@ -725,11 +725,6 @@ def _add_COLR(font, model, master_fonts, axisTags, colr_layer_reuse=True): varIdxes = [mapping[v] for v in merger.varIdxes] colr.VarIndexMap = builder.buildDeltaSetIndexMap(varIdxes) - # rebuild LayerList to optimize PaintColrLayers layer reuse - if colr.LayerList and colr_layer_reuse: - colorGlyphs = unbuildColrV1(colr.LayerList, colr.BaseGlyphList) - colr.LayerList, colr.BaseGlyphList = buildColrV1(colorGlyphs, allowLayerReuse=True) - def load_designspace(designspace): # TODO: remove this and always assume 'designspace' is a DesignSpaceDocument, diff --git a/Lib/fontTools/varLib/merger.py b/Lib/fontTools/varLib/merger.py index f4ab29d16..358e9df82 100644 --- a/Lib/fontTools/varLib/merger.py +++ b/Lib/fontTools/varLib/merger.py @@ -4,10 +4,13 @@ Merge OpenType Layout tables (GDEF / GPOS / GSUB). import os import copy import enum +import itertools from operator import ior import logging +from fontTools.colorLib.builder import MAX_PAINT_COLR_LAYER_COUNT, LayerReuseCache from fontTools.misc import classifyTools from fontTools.misc.roundTools import otRound +from fontTools.misc.treeTools import build_n_ary_tree from fontTools.ttLib.tables import otTables as ot from fontTools.ttLib.tables import otBase as otBase from fontTools.ttLib.tables.otConverters import BaseFixedValue @@ -1131,7 +1134,7 @@ class COLRVariationMerger(VariationMerger): care of that too. """ - def __init__(self, model, axisTags, font): + def __init__(self, model, axisTags, font, allowLayerReuse=True): VariationMerger.__init__(self, model, axisTags, font) # maps {tuple(varIdxes): VarIndexBase} to facilitate reuse of VarIndexBase # between variable tables with same varIdxes. @@ -1141,6 +1144,14 @@ class COLRVariationMerger(VariationMerger): # set of id()s of the subtables that contain variations after merging # and need to be upgraded to the associated VarType. self.varTableIds = set() + # we keep these around for rebuilding a LayerList while merging PaintColrLayers + self.layers = [] + self.uniqueLayerIDs = set() + self.layerReuseCache = None + if allowLayerReuse: + self.layerReuseCache = LayerReuseCache() + # flag to ensure BaseGlyphList is fully merged before LayerList gets processed + self._doneBaseGlyphs = False def mergeTables(self, font, master_ttfs, tableTags=("COLR",)): VariationMerger.mergeTables(self, font, master_ttfs, tableTags) @@ -1281,9 +1292,126 @@ class COLRVariationMerger(VariationMerger): setattr(parent, st.name, newSubTable) +@COLRVariationMerger.merger(ot.BaseGlyphList) +def merge(merger, self, lst): + # ignore BaseGlyphCount, allow sparse glyph sets across masters + out = {rec.BaseGlyph: rec for rec in self.BaseGlyphPaintRecord} + masters = [{rec.BaseGlyph: rec for rec in m.BaseGlyphPaintRecord} for m in lst] + + for i, g in enumerate(out.keys()): + try: + # missing base glyphs don't participate in the merge + merger.mergeThings(out[g], [v.get(g) for v in masters]) + except VarLibMergeError as e: + e.stack.append(f".BaseGlyphPaintRecord[{i}]") + e.cause["location"] = f"base glyph {g!r}" + raise + + merger._doneBaseGlyphs = True + + +@COLRVariationMerger.merger(ot.LayerList) +def merge(merger, self, lst): + # nothing to merge for LayerList, assuming we have already merged all PaintColrLayers + # found while traversing the paint graphs rooted at BaseGlyphPaintRecords. + assert merger._doneBaseGlyphs, "BaseGlyphList must be merged before LayerList" + # Simply flush the final list of layers and go home. + self.LayerCount = len(merger.layers) + self.Paint = merger.layers + + +def _flatten_layers(paint, colr): + if paint.Format == ot.PaintFormat.PaintColrLayers: + yield from itertools.chain( + *(_flatten_layers(l, colr) for l in paint.getChildren(colr)) + ) + else: + yield paint + + +def _merge_PaintColrLayers(self, out, lst): + # we only enforce that the (flat) number of layers is the same across all masters + # but we allow FirstLayerIndex to differ to acommodate for sparse glyph sets. + out_layers = [] + for paint in _flatten_layers(out, self.font["COLR"].table): + if id(paint) in self.uniqueLayerIDs: + # ensure dest paints are unique, since merging operation modifies in-place + paint2 = copy.deepcopy(paint) + assert id(paint2) not in self.uniqueLayerIDs + paint = paint2 + out_layers.append(paint) + + # sanity check ttfs are subset to current values (see VariationMerger.mergeThings) + # before matching each master PaintColrLayers to its respective COLR by position + assert len(self.ttfs) == len(lst) + master_layerses = [ + list(_flatten_layers(lst[i], self.ttfs[i]["COLR"].table)) + for i in range(len(lst)) + ] + + try: + self.mergeLists(out_layers, master_layerses) + except VarLibMergeError as e: + # NOTE: This attribute doesn't actually exist in PaintColrLayers but it's + # handy to have it in the stack trace for debugging. + e.stack.append(".Layers") + raise + + # following block is very similar to LayerListBuilder._beforeBuildPaintColrLayers + # but I couldn't find a nice way to share the code between the two... + + if self.layerReuseCache is not None: + # successful reuse can make the list smaller + out_layers = self.layerReuseCache.try_reuse(out_layers) + + # if the list is still too big we need to tree-fy it + is_tree = len(out_layers) > MAX_PAINT_COLR_LAYER_COUNT + out_layers = build_n_ary_tree(out_layers, n=MAX_PAINT_COLR_LAYER_COUNT) + + # We now have a tree of sequences with Paint leaves. + # Convert the sequences into PaintColrLayers. + def listToColrLayers(paint): + if isinstance(paint, list): + layers = [listToColrLayers(l) for l in paint] + paint = ot.Paint() + paint.Format = int(ot.PaintFormat.PaintColrLayers) + paint.NumLayers = len(layers) + paint.FirstLayerIndex = len(self.layers) + self.layers.exend(layers) + if self.layerReuseCache is not None: + self.layerReuseCache.add(layers, paint.FirstLayerIndex) + return paint + + out_layers = [listToColrLayers(l) for l in out_layers] + + if len(out_layers) == 1 and out_layers[0].Format == ot.PaintFormat.PaintColrLayers: + # special case when the reuse cache finds a single perfect PaintColrLayers match + # (it can only come from a successful reuse, _flatten_layers has gotten rid of + # all nested PaintColrLayers already); we assign it directly and avoid creating + # an extra table + out.NumLayers = out_layers[0].NumLayers + out.FirstLayerIndex = out_layers[0].FirstLayerIndex + else: + out.NumLayers = len(out_layers) + out.FirstLayerIndex = len(self.layers) + + self.layers.extend(out_layers) + self.uniqueLayerIDs.update(id(p) for p in out_layers) + + # Register our parts for reuse provided we aren't a tree + # If we are a tree the leaves registered for reuse and that will suffice + if self.layerReuseCache is not None and not is_tree: + self.layerReuseCache.add(out_layers, out.FirstLayerIndex) + + @COLRVariationMerger.merger((ot.Paint, ot.ClipBox)) def merge(merger, self, lst): fmt = merger.checkFormatEnum(self, lst, lambda fmt: not fmt.is_variable()) + + if fmt is ot.PaintFormat.PaintColrLayers: + _merge_PaintColrLayers(merger, self, lst) + return + varFormat = fmt.as_variable() varAttrs = () diff --git a/Tests/varLib/merger_test.py b/Tests/varLib/merger_test.py index 23c416d45..fe511730b 100644 --- a/Tests/varLib/merger_test.py +++ b/Tests/varLib/merger_test.py @@ -1,6 +1,6 @@ from copy import deepcopy import string -from fontTools.colorLib.builder import LayerListBuilder, buildClipList +from fontTools.colorLib.builder import LayerListBuilder, buildCOLR, buildClipList from fontTools.misc.testTools import getXML from fontTools.varLib.merger import COLRVariationMerger from fontTools.varLib.models import VariationModel @@ -24,6 +24,8 @@ def dump_xml(table, ttFont=None): def compile_decompile(table, ttFont): writer = OTTableWriter(tableTag="COLR") + # compile itself may modify a table, safer to copy it first + table = deepcopy(table) table.compile(writer, ttFont) data = writer.getAllData() @@ -785,3 +787,550 @@ class COLRVariationMergerTest: 1, 1, ] + + @pytest.mark.parametrize( + "color_glyphs, reuse, expected_xml, expected_varIdxes", + [ + pytest.param( + [ + { + "A": { + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 0, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + }, + { + "A": { + "Format": ot.PaintFormat.PaintColrLayers, + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 0, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + }, + ], + False, + [ + "", + ' ', + " ", + " ", + " ", + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + " ", + " ", + " ", + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + " ", + "", + ], + [], + id="no-variation", + ), + pytest.param( + [ + { + "A": { + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 0, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + "C": { + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 3, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + }, + { + # NOTE: 'A' is missing from non-default master + "C": { + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 3, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + ], + }, + }, + ], + False, + [ + "", + ' ', + " ", + " ", + " ", + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + " ", + " ", + " ", + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + " ", + "", + ], + [0], + id="sparse-masters", + ), + pytest.param( + [ + { + "A": { + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 0, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + "C": { + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + # 'C' reuses layers 1-3 from 'A' + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + "D": { # identical to 'C' + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + "E": { # superset of 'C' or 'D' + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 3, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + }, + { + # NOTE: 'A' is missing from non-default master + "C": { + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + ], + }, + "D": { # same as 'C' + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + ], + }, + "E": { # first two layers vary the same way as 'C' or 'D' + "Format": int(ot.PaintFormat.PaintColrLayers), + "Layers": [ + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 1, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 2, + "Alpha": 0.5, + }, + "Glyph": "B", + }, + { + "Format": int(ot.PaintFormat.PaintGlyph), + "Paint": { + "Format": int(ot.PaintFormat.PaintSolid), + "PaletteIndex": 3, + "Alpha": 1.0, + }, + "Glyph": "B", + }, + ], + }, + }, + ], + True, # reuse + [ + "", + ' ', + " ", + " ", + " ", + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + " ", + " ", + " ", + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + ' ', + ' ', + ' ', + " ", + ' ', + ' ', + ' ', + ' ', + " ", + ' ', + " ", + " ", + "", + ], + [0], + id="sparse-masters-with-reuse", + ), + ], + ) + def test_merge_full_table( + self, color_glyphs, ttFont, expected_xml, expected_varIdxes, reuse + ): + master_ttfs = [deepcopy(ttFont) for _ in range(len(color_glyphs))] + for ttf, glyphs in zip(master_ttfs, color_glyphs): + # merge algorithm is expected to work even if the master COLRs may differ as + # to the layer reuse, hence we force this is on while building them (even + # if it's on by default anyway, we want to make sure it works under more + # complex scenario). + ttf["COLR"] = buildCOLR(glyphs, allowLayerReuse=True) + vf = deepcopy(master_ttfs[0]) + + model = VariationModel([{}, {"ZZZZ": 1.0}]) + merger = COLRVariationMerger(model, ["ZZZZ"], vf, allowLayerReuse=reuse) + + merger.mergeTables(vf, master_ttfs) + + out = vf["COLR"].table + + assert compile_decompile(out, vf) == out + assert dump_xml(out, vf) == expected_xml + assert merger.varIdxes == expected_varIdxes