shadowfacts/v6
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use pyftsubset for font subsetting

Browse Source
This commit is contained in:
Shadowfacts 2025-02-11 14:10:13 -05:00
parent 1629a7e30c
commit 6f6e1453f7
11 changed files with 392 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "crates/pyftsubset/fonttools"]
path = crates/pyftsubset/fonttools
url = https://git.shadowfacts.net/shadowfacts/fonttools.git

216
Cargo.lock generated
View File

@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"getrandom",
"getrandom 0.2.15",
"once_cell",
"version_check",
"zerocopy",
@ -155,6 +155,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bit-vec"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]]
name = "bitflags"
version = "1.3.2"
@ -517,6 +523,22 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
dependencies = [
"libc",
"windows-sys 0.59.0",
]
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "filetime"
version = "0.2.25"
@ -683,10 +705,22 @@ dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi",
"wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
]
[[package]]
name = "getrandom"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
dependencies = [
"cfg-if",
"libc",
"wasi 0.13.3+wasi-0.2.2",
"windows-targets",
]
[[package]]
name = "gimli"
version = "0.31.1"
@ -722,7 +756,7 @@ name = "grass"
version = "0.13.4"
source = "git+https://git.shadowfacts.net/shadowfacts/grass.git?branch=custom-global-variables#e64e648b6174d23b6d4a8ad674eb443dc6fcbdff"
dependencies = [
"getrandom",
"getrandom 0.2.15",
"grass_compiler",
]
@ -749,6 +783,12 @@ dependencies = [
"allocator-api2",
]
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "html5ever"
version = "0.27.0"
@ -935,6 +975,12 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "indoc"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"
[[package]]
name = "inotify"
version = "0.10.2"
@ -1044,6 +1090,12 @@ dependencies = [
"redox_syscall",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "lock_api"
version = "0.4.12"
@ -1056,9 +1108,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.22"
version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
[[package]]
name = "mac"
@ -1098,6 +1150,15 @@ version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "memoffset"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -1131,7 +1192,7 @@ checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
dependencies = [
"libc",
"log",
"wasi",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.52.0",
]
@ -1195,9 +1256,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.20.2"
version = "1.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e"
[[package]]
name = "parking_lot"
@ -1375,6 +1436,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "portable-atomic"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
[[package]]
name = "ppv-lite86"
version = "0.2.20"
@ -1418,6 +1485,78 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
[[package]]
name = "pyftsubset"
version = "0.1.0"
dependencies = [
"log",
"pyo3",
"tempfile",
]
[[package]]
name = "pyo3"
version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc"
dependencies = [
"cfg-if",
"indoc",
"libc",
"memoffset",
"once_cell",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
"pyo3-macros",
"unindent",
]
[[package]]
name = "pyo3-build-config"
version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7"
dependencies = [
"once_cell",
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d"
dependencies = [
"libc",
"pyo3-build-config",
]
[[package]]
name = "pyo3-macros"
version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
"quote",
"syn",
]
[[package]]
name = "pyo3-macros-backend"
version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4"
dependencies = [
"heck",
"proc-macro2",
"pyo3-build-config",
"quote",
"syn",
]
[[package]]
name = "quick-xml"
version = "0.37.2"
@ -1464,7 +1603,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
"getrandom 0.2.15",
]
[[package]]
@ -1523,6 +1662,19 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags 2.7.0",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
]
[[package]]
name = "ryu"
version = "1.0.18"
@ -1727,6 +1879,26 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
[[package]]
name = "target-lexicon"
version = "0.12.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
[[package]]
name = "tempfile"
version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91"
dependencies = [
"cfg-if",
"fastrand",
"getrandom 0.3.1",
"once_cell",
"rustix",
"windows-sys 0.59.0",
]
[[package]]
name = "tendril"
version = "0.4.3"
@ -2220,6 +2392,12 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
[[package]]
name = "unindent"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
[[package]]
name = "utf-8"
version = "0.7.6"
@ -2239,6 +2417,7 @@ dependencies = [
"ahash",
"anyhow",
"base64",
"bit-vec",
"bitflags 2.7.0",
"chrono",
"clap",
@ -2261,6 +2440,7 @@ dependencies = [
"once_cell",
"pulldown-cmark",
"pulldown-cmark-escape",
"pyftsubset",
"regex",
"rss",
"serde",
@ -2308,6 +2488,15 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasi"
version = "0.13.3+wasi-0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
dependencies = [
"wit-bindgen-rt",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.99"
@ -2471,6 +2660,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "wit-bindgen-rt"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
dependencies = [
"bitflags 2.7.0",
]
[[package]]
name = "xml5ever"
version = "0.18.1"

3
Cargo.toml
View File

@ -2,6 +2,7 @@ workspace = { members = [
"crates/compute_graph",
"crates/compute_graph_macros",
"crates/derive_test",
"crates/pyftsubset",
] }
[package]
@ -19,6 +20,7 @@ serde_json = "1.0"
ahash = "0.8.11"
anyhow = "1.0.95"
base64 = "0.22.1"
bit-vec = "0.8.0"
bitflags = "2.7.0"
chrono = { version = "0.4.39", features = ["serde"] }
clap = { version = "4.5.23", features = ["cargo"] }
@ -43,6 +45,7 @@ notify = "7.0.0"
once_cell = "1.20.2"
pulldown-cmark = "0.12.2"
pulldown-cmark-escape = "0.11.0"
pyftsubset = { path = "crates/pyftsubset" }
regex = "1.11.1"
rss = { version = "2.0.11", features = ["atom"] }
serde = { version = "1.0", features = ["derive"] }

9
crates/pyftsubset/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "pyftsubset"
version = "0.1.0"
edition = "2024"
[dependencies]
log = "0.4.25"
pyo3 = "0.23.4"
tempfile = "3.16.0"

1
crates/pyftsubset/fonttools Submodule

@ -0,0 +1 @@
Subproject commit d6f40c2e453f3c07807fef7926a31717f180b660

65
crates/pyftsubset/src/lib.rs Normal file
View File

@ -0,0 +1,65 @@
use std::{
ffi::CString,
fs::read_to_string,
io::{Read, Write},
path::Path,
};
use log::debug;
use pyo3::{
ffi::c_str,
prelude::*,
types::{PyList, PyTuple},
};
use tempfile::NamedTempFile;
pub fn subset(data: &[u8], unicodes: &[u32]) -> Vec<u8> {
pyo3::prepare_freethreaded_python();
let mut input = NamedTempFile::new().expect("input file");
input.write_all(data).expect("writing input");
let mut output = NamedTempFile::new().expect("output file");
let path = Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/fonttools/Lib"));
let init_path = path.join("fontTools/subset/__init__.py");
let py_init = CString::new(read_to_string(init_path).unwrap()).unwrap();
let result = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
let syspath = py
.import("sys")?
.getattr("path")?
.downcast_into::<PyList>()?;
syspath.insert(0, path)?;
let app: Py<PyAny> = PyModule::from_code(py, py_init.as_c_str(), c_str!(""), c_str!(""))?
.getattr("main")?
.into();
let args = PyList::new(py, vec![
input.path().to_str().unwrap(),
&format_unicodes(unicodes),
&format!("--output-file={}", output.path().to_str().unwrap()),
])?;
debug!("Running pyftsubset with {:?}", args);
let args_tuple = PyTuple::new(py, &[args])?;
app.call1(py, args_tuple)
});
result.expect("subsetting");
let mut buf = vec![];
output.read_to_end(&mut buf).expect("reading output");
buf
}
fn format_unicodes(unicodes: &[u32]) -> String {
use std::fmt::Write;
let mut s = "--unicodes=".to_owned();
let mut first = true;
for u in unicodes {
if first {
write!(s, "{:x}", u).expect("append unicode");
first = false;
} else {
write!(s, ",{:x}", u).expect("append unicode");
}
}
s
}

8
site_test/css/fonts.scss
View File

@ -41,9 +41,17 @@
font-weight: normal;
font-style: normal;
}
@font-face {
font-family: "Berkeley Mono";
src: url("data:font/woff2;base64," + $berkeley-mono-italic) format("woff2");
font-weight: normal;
font-style: italic;
}
@font-face {
font-family: "Berkeley Mono";
src: url("data:font/woff2;base64," + $berkeley-mono-bold) format("woff2");
font-weight: bold;
font-style: normal;
}

BIN
site_test/css/fonts/BerkeleyMono-Bold.woff2 Normal file
View File

Binary file not shown.

12
src/generator/css.rs
View File

@ -46,7 +46,7 @@ pub fn make_graph(
);
let character_sets = character_sets::make_graph(builder, render_inputs, render_dynamic_inputs);
let assertion = builder.add_rule(AssertNoBoldMonospace(character_sets.clone()));
let assertion = builder.add_rule(AssertNoBoldItalicMonospace(character_sets.clone()));
let fonts = [
(
@ -79,6 +79,11 @@ pub fn make_graph(
content_path("css/fonts/BerkeleyMono-Oblique.woff2"),
FontKey::MONOSPACE.union(FontKey::ITALIC),
),
(
"berkeley-mono-bold",
content_path("css/fonts/BerkeleyMono-Bold.woff2"),
FontKey::MONOSPACE.union(FontKey::BOLD),
),
];
for (name, path, font_key) in fonts.into_iter() {
let file = read_file(path, builder, &mut *watcher_);
@ -223,12 +228,11 @@ impl Rule for ConvertToBase64 {
}
#[derive(InputVisitable)]
struct AssertNoBoldMonospace(Input<CharacterSets>);
impl Rule for AssertNoBoldMonospace {
struct AssertNoBoldItalicMonospace(Input<CharacterSets>);
impl Rule for AssertNoBoldItalicMonospace {
type Output = ();
fn evaluate(&mut self) -> Self::Output {
let sets = self.input_0();
assert!(sets.bold_monospace().is_empty());
assert!(sets.bold_italic_monospace().is_empty());
}
}

104
src/generator/css/character_sets.rs
View File

@ -1,3 +1,4 @@
use bit_vec::BitVec;
use bitflags::bitflags;
use compute_graph::{
NodeId,
@ -9,7 +10,7 @@ use compute_graph::{
use html5ever::{
Attribute, local_name,
tokenizer::{
BufferQueue, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
BufferQueue, Tag, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
TokenizerResult,
},
};
@ -105,7 +106,7 @@ impl Rule for UnionDynamicCharacterSets {
}
fn get_character_sets(html: &str) -> CharacterSets {
let accumulator = CharacterSetAccumulator::default();
let accumulator = CharacterSetAccumulator::new();
let mut tokenizer = Tokenizer::new(accumulator, TokenizerOpts::default());
let mut queue = BufferQueue::default();
queue.push_back(html.into());
@ -205,7 +206,7 @@ impl std::fmt::Debug for CharacterSets {
}
bitflags! {
#[derive(Clone, Copy)]
#[derive(Clone, Copy, PartialEq)]
pub struct FontKey: u8 {
const BOLD = 1;
const ITALIC = 1 << 1;
@ -215,41 +216,72 @@ bitflags! {
const FONT_VARIATIONS: usize = FontKey::all().bits() as usize + 1;
#[derive(Default)]
struct FontKeyStack {
bold: BitVec,
italic: BitVec,
monospace: BitVec,
}
impl FontKeyStack {
fn push(&mut self, key: FontKey) {
self.bold.push(key.contains(FontKey::BOLD));
self.italic.push(key.contains(FontKey::ITALIC));
self.monospace.push(key.contains(FontKey::MONOSPACE));
}
fn pop(&mut self) {
self.bold.pop();
self.italic.pop();
self.monospace.pop();
}
fn all_ored(&self) -> FontKey {
let mut all = FontKey::empty();
if self.bold.any() {
all.insert(FontKey::BOLD);
}
if self.italic.any() {
all.insert(FontKey::ITALIC);
}
if self.monospace.any() {
all.insert(FontKey::MONOSPACE);
}
all
}
}
// N.B.: html5ever 0.28.0 changed TokenSink::process_token to take &self.
// At which point we upgrade, the state on this type will need to use something
// else to provide interior mutability.
#[derive(Default)]
struct CharacterSetAccumulator {
characters: CharacterSets,
depths: [usize; 3],
keys: FontKeyStack,
}
impl CharacterSetAccumulator {
fn new() -> Self {
assert_eq!(FontKey::BOLD.bits().trailing_zeros(), 0);
assert_eq!(FontKey::ITALIC.bits().trailing_zeros(), 1);
assert_eq!(FontKey::MONOSPACE.bits().trailing_zeros(), 2);
Self {
characters: CharacterSets::default(),
keys: FontKeyStack::default(),
}
}
fn handle_token(&mut self, token: Token) {
match token {
Token::TagToken(tag) => {
let depth = if tag.name == local_name!("strong") || tag.name == local_name!("b") {
&mut self.depths[0]
} else if tag.name == local_name!("em")
|| tag.name == local_name!("i")
|| tag.attrs.iter().any(Self::is_hl_cmt)
{
&mut self.depths[1]
} else if tag.name == local_name!("code") {
&mut self.depths[2]
} else {
return;
};
if tag.kind == TagKind::StartTag {
*depth += 1;
let key = Self::font_key_for(&tag);
self.keys.push(key);
} else {
*depth -= 1;
self.keys.pop();
}
}
Token::CharacterTokens(s) => {
let mut key = FontKey::empty();
key.set(FontKey::BOLD, self.depths[0] > 0);
key.set(FontKey::ITALIC, self.depths[1] > 0);
key.set(FontKey::MONOSPACE, self.depths[2] > 0);
let key = self.keys.all_ored();
let set = self.characters.get_mut(key);
set.extend(s.chars());
}
@ -257,6 +289,32 @@ impl CharacterSetAccumulator {
}
}
fn font_key_for(tag: &Tag) -> FontKey {
if tag.name == local_name!("strong") || tag.name == local_name!("b") || Self::is_header(tag)
{
FontKey::BOLD
} else if tag.name == local_name!("em")
|| tag.name == local_name!("i")
|| tag.name == local_name!("header")
|| tag.attrs.iter().any(Self::is_hl_cmt)
{
FontKey::ITALIC
} else if tag.name == local_name!("code") {
FontKey::MONOSPACE
} else {
FontKey::empty()
}
}
fn is_header(tag: &Tag) -> bool {
tag.name == local_name!("h1")
|| tag.name == local_name!("h2")
|| tag.name == local_name!("h3")
|| tag.name == local_name!("h4")
|| tag.name == local_name!("h5")
|| tag.name == local_name!("h6")
}
fn is_hl_cmt(attr: &Attribute) -> bool {
attr.name.prefix == None
&& attr.name.local == local_name!("class")

8
src/generator/css/font_subset.rs
View File

@ -4,6 +4,7 @@ use compute_graph::{
rule::Rule,
synchronicity::Asynchronous,
};
use pyftsubset::subset;
use super::character_sets::{CharacterSets, FontKey};
@ -41,6 +42,11 @@ struct SubsetFont {
impl Rule for SubsetFont {
type Output = Vec<u8>;
fn evaluate(&mut self) -> Self::Output {
self.font().clone()
let unicodes = self
.characters()
.iter()
.map(|&c| c as u32)
.collect::<Vec<_>>();
subset(self.font().as_ref(), &unicodes)
}
}