From 1629a7e30c6010812f7412f99ade448f7d0d94df Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Tue, 14 Jan 2025 21:23:11 -0500 Subject: [PATCH] Character set extraction --- Cargo.lock | 19 +- Cargo.toml | 2 + src/generator/archive.rs | 2 +- src/generator/css.rs | 92 ++++++-- src/generator/css/character_sets.rs | 322 ++++++++++++++++++++++++++++ src/generator/css/font_subset.rs | 46 ++++ src/generator/home.rs | 2 +- src/generator/mod.rs | 50 +++-- src/generator/not_found.rs | 8 +- src/generator/posts.rs | 14 +- src/generator/tags.rs | 11 +- src/generator/templates.rs | 22 +- src/generator/tutorials.rs | 32 +-- src/generator/tv.rs | 10 +- 14 files changed, 537 insertions(+), 95 deletions(-) create mode 100644 src/generator/css/character_sets.rs create mode 100644 src/generator/css/font_subset.rs diff --git a/Cargo.lock b/Cargo.lock index 5458bf5..96feda4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -162,9 +163,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" [[package]] name = "block-buffer" @@ -711,7 +712,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "ignore", "walkdir", ] @@ -1038,7 +1039,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "libc", "redox_syscall", ] @@ -1152,7 +1153,7 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "filetime", "fsevent-sys", "inotify", @@ -1404,7 +1405,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "getopts", "memchr", "pulldown-cmark-escape", @@ -1472,7 +1473,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", ] [[package]] @@ -1933,7 +1934,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "bytes", "futures-util", "http", @@ -2235,8 +2236,10 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" name = "v7" version = "0.1.0" dependencies = [ + "ahash", "anyhow", "base64", + "bitflags 2.7.0", "chrono", "clap", "compute_graph", diff --git a/Cargo.toml b/Cargo.toml index 4cdd027..c48a7bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,10 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" [dependencies] +ahash = "0.8.11" anyhow = "1.0.95" base64 = "0.22.1" +bitflags = "2.7.0" chrono = { version = "0.4.39", features = ["serde"] } clap = { version = "4.5.23", features = ["cargo"] } compute_graph = { path = "crates/compute_graph" } diff --git a/src/generator/archive.rs b/src/generator/archive.rs index 88da2ce..1cda671 100644 --- a/src/generator/archive.rs +++ b/src/generator/archive.rs @@ -23,7 +23,7 @@ pub fn make_graph( posts: Input>>, default_template: Input, watcher: &mut FileWatcher, -) -> Input<()> { +) -> Input { let entries = builder.add_rule(Entries(posts)); let posts_by_year = builder.add_rule(PostsByYear(entries)); diff --git a/src/generator/css.rs b/src/generator/css.rs index 279ce89..aa4a108 100644 --- a/src/generator/css.rs +++ b/src/generator/css.rs @@ -1,3 +1,6 @@ +mod character_sets; +mod font_subset; + use std::{ cell::RefCell, collections::{HashMap, HashSet}, @@ -7,10 +10,11 @@ use std::{ }; use base64::{Engine, prelude::BASE64_STANDARD}; +use character_sets::{CharacterSets, FontKey}; use compute_graph::{ InvalidationSignal, builder::GraphBuilder, - input::{Input, InputVisitable, InputVisitor}, + input::{DynamicInput, Input, InputVisitable, InputVisitor}, rule::Rule, synchronicity::Asynchronous, }; @@ -20,44 +24,67 @@ use log::error; use super::{ FileWatcher, - util::{content_path, output_writer}, + util::{Combine, content_path, output_writer}, }; pub fn make_graph( builder: &mut GraphBuilder<(), Asynchronous>, + render_inputs: Vec>, + render_dynamic_inputs: Vec>, watcher: Rc>, ) -> Input<()> { let mut watcher_ = watcher.borrow_mut(); - let mut files = HashMap::<&'static str, Input>::new(); - let filenames = [ + let mut variables = HashMap::<&'static str, Input>::new(); + + variables.insert( + "external-link", + read_to_base64( + content_path("css/external-link.svg"), + builder, + &mut *watcher_, + ), + ); + + let character_sets = character_sets::make_graph(builder, render_inputs, render_dynamic_inputs); + let assertion = builder.add_rule(AssertNoBoldMonospace(character_sets.clone())); + + let fonts = [ ( "valkyrie-a-regular", content_path("css/fonts/valkyrie_a_regular.woff2"), + FontKey::empty(), ), ( "valkyrie-a-bold", content_path("css/fonts/valkyrie_a_bold.woff2"), + FontKey::BOLD, ), ( "valkyrie-a-italic", content_path("css/fonts/valkyrie_a_italic.woff2"), + FontKey::ITALIC, ), ( "valkyrie-a-bold-italic", content_path("css/fonts/valkyrie_a_bold_italic.woff2"), + FontKey::BOLD.union(FontKey::ITALIC), ), - ("external-link", content_path("css/external-link.svg")), ( "berkeley-mono-regular", content_path("css/fonts/BerkeleyMono-Regular.woff2"), + FontKey::MONOSPACE, ), ( "berkeley-mono-italic", content_path("css/fonts/BerkeleyMono-Oblique.woff2"), + FontKey::MONOSPACE.union(FontKey::ITALIC), ), ]; - for (name, path) in filenames.into_iter() { - files.insert(name, read_file(path, builder, &mut *watcher_)); + for (name, path, font_key) in fonts.into_iter() { + let file = read_file(path, builder, &mut *watcher_); + let subsetted = font_subset::make_graph(builder, file, character_sets.clone(), font_key); + let base64 = builder.add_rule(ConvertToBase64(subsetted)); + variables.insert(name, base64); } drop(watcher_); @@ -66,31 +93,40 @@ pub fn make_graph( watcher, watched: HashSet::new(), invalidate: Rc::clone(&invalidate_css_box), - fonts: files, + variables, }); invalidate_css_box.replace(Some(invalidate_css)); - css + builder.add_rule(Combine(css, assertion)) } fn read_file( path: PathBuf, builder: &mut GraphBuilder<(), Asynchronous>, watcher: &mut FileWatcher, -) -> Input { - let (font, invalidate) = builder.add_invalidatable_rule(ReadFile(path.clone())); +) -> Input> { + let (file, invalidate) = builder.add_invalidatable_rule(ReadFile(path.clone())); watcher.watch(path, move || invalidate.invalidate()); - font + file +} + +fn read_to_base64( + path: PathBuf, + builder: &mut GraphBuilder<(), Asynchronous>, + watcher: &mut FileWatcher, +) -> Input { + let file = read_file(path, builder, watcher); + builder.add_rule(ConvertToBase64(file)) } struct CompileScss { watcher: Rc>, watched: HashSet, invalidate: Rc>>, - fonts: HashMap<&'static str, Input>, + variables: HashMap<&'static str, Input>, } impl InputVisitable for CompileScss { fn visit_inputs(&self, visitor: &mut impl InputVisitor) { - for input in self.fonts.values() { + for input in self.variables.values() { visitor.visit(input); } } @@ -106,7 +142,7 @@ impl Rule for CompileScss { OutputStyle::Compressed }; let mut options = Options::default().fs(&fs).style(style); - for (name, input) in self.fonts.iter() { + for (name, input) in self.variables.iter() { let value = Value::String(input.value().to_owned(), QuoteKind::None); options = options.add_custom_var(*name, value); } @@ -160,14 +196,14 @@ impl<'a> Fs for TrackingFs<'a> { #[derive(InputVisitable)] struct ReadFile(#[skip_visit] PathBuf); impl Rule for ReadFile { - type Output = String; + type Output = Vec; fn evaluate(&mut self) -> Self::Output { match std::fs::read(&self.0) { - Ok(data) => BASE64_STANDARD.encode(data), + Ok(data) => data, Err(e) => { error!("Error reading font {:?}: {:?}", &self.0, e); - String::new() + vec![] } } } @@ -176,3 +212,23 @@ impl Rule for ReadFile { write!(f, "{}", self.0.display()) } } + +#[derive(InputVisitable)] +struct ConvertToBase64(Input>); +impl Rule for ConvertToBase64 { + type Output = String; + fn evaluate(&mut self) -> Self::Output { + BASE64_STANDARD.encode(self.input_0().as_slice()) + } +} + +#[derive(InputVisitable)] +struct AssertNoBoldMonospace(Input); +impl Rule for AssertNoBoldMonospace { + type Output = (); + fn evaluate(&mut self) -> Self::Output { + let sets = self.input_0(); + assert!(sets.bold_monospace().is_empty()); + assert!(sets.bold_italic_monospace().is_empty()); + } +} diff --git a/src/generator/css/character_sets.rs b/src/generator/css/character_sets.rs new file mode 100644 index 0000000..0befa6f --- /dev/null +++ b/src/generator/css/character_sets.rs @@ -0,0 +1,322 @@ +use bitflags::bitflags; +use compute_graph::{ + NodeId, + builder::GraphBuilder, + input::{DynamicInput, Input, InputVisitable}, + rule::{DynamicNodeFactory, DynamicRule, DynamicRuleContext, Rule}, + synchronicity::Asynchronous, +}; +use html5ever::{ + Attribute, local_name, + tokenizer::{ + BufferQueue, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts, + TokenizerResult, + }, +}; + +pub fn make_graph( + builder: &mut GraphBuilder<(), Asynchronous>, + inputs: Vec>, + dynamic_inputs: Vec>, +) -> Input { + let mut charsets = inputs + .into_iter() + .map(|inp| builder.add_rule(BuildCharacterSet(inp))) + .collect::>(); + let dynamic_charsets = + builder.add_dynamic_rule(MakeBuildDynamicCharacterSets::new(dynamic_inputs)); + let dynamic_unioned = builder.add_rule(UnionDynamicCharacterSets(dynamic_charsets)); + charsets.push(dynamic_unioned); + builder.add_rule(UnionCharacterSets(charsets)) +} + +#[derive(InputVisitable)] +struct BuildCharacterSet(Input); +impl Rule for BuildCharacterSet { + type Output = CharacterSets; + fn evaluate(&mut self) -> Self::Output { + get_character_sets(&*self.input_0()) + } +} + +struct MakeBuildDynamicCharacterSets { + strings: Vec>, + factory: DynamicNodeFactory, +} +impl MakeBuildDynamicCharacterSets { + fn new(strings: Vec>) -> Self { + Self { + strings, + factory: DynamicNodeFactory::new(), + } + } +} +impl InputVisitable for MakeBuildDynamicCharacterSets { + fn visit_inputs(&self, visitor: &mut impl compute_graph::input::InputVisitor) { + for input in self.strings.iter() { + input.visit_inputs(visitor); + } + } +} +impl DynamicRule for MakeBuildDynamicCharacterSets { + type ChildOutput = CharacterSets; + fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { + for dynamic_input in self.strings.iter() { + for input in dynamic_input.value().inputs.iter() { + self.factory.add_node(ctx, input.node_id(), |ctx| { + ctx.add_rule(BuildCharacterSet(input.clone())) + }); + } + } + self.factory.all_nodes(ctx) + } +} + +struct UnionCharacterSets(Vec>); +impl InputVisitable for UnionCharacterSets { + fn visit_inputs(&self, visitor: &mut impl compute_graph::input::InputVisitor) { + for input in self.0.iter() { + input.visit_inputs(visitor); + } + } +} +impl Rule for UnionCharacterSets { + type Output = CharacterSets; + fn evaluate(&mut self) -> Self::Output { + let mut merged = CharacterSets::default(); + for input in self.0.iter() { + merged.extend(&*input.value()); + } + merged + } +} + +#[derive(InputVisitable)] +struct UnionDynamicCharacterSets(DynamicInput); +impl Rule for UnionDynamicCharacterSets { + type Output = CharacterSets; + fn evaluate(&mut self) -> Self::Output { + let mut merged = CharacterSets::default(); + for input in self.0.value().inputs.iter() { + merged.extend(&*input.value()); + } + merged + } +} + +fn get_character_sets(html: &str) -> CharacterSets { + let accumulator = CharacterSetAccumulator::default(); + let mut tokenizer = Tokenizer::new(accumulator, TokenizerOpts::default()); + let mut queue = BufferQueue::default(); + queue.push_back(html.into()); + let result = tokenizer.feed(&mut queue); + match result { + TokenizerResult::Done => (), + result => panic!("unexpected result: {result:?}"), + } + tokenizer.sink.characters +} + +#[derive(Default, PartialEq, Clone)] +pub struct CharacterSets { + sets: [ahash::HashSet; FONT_VARIATIONS], +} +impl CharacterSets { + pub fn get(&self, key: FontKey) -> &ahash::HashSet { + &self.sets[key.bits() as usize] + } + + pub fn get_mut(&mut self, key: FontKey) -> &mut ahash::HashSet { + &mut self.sets[key.bits() as usize] + } + + fn extend(&mut self, other: &CharacterSets) { + for i in 0..FONT_VARIATIONS { + self.sets[i].extend(&other.sets[i]); + } + } +} + +#[allow(unused)] +impl CharacterSets { + pub fn regular(&self) -> &ahash::HashSet { + &self.sets[0] + } + + pub fn bold(&self) -> &ahash::HashSet { + self.get(FontKey::BOLD) + } + + pub fn italic(&self) -> &ahash::HashSet { + self.get(FontKey::ITALIC) + } + + pub fn monospace(&self) -> &ahash::HashSet { + self.get(FontKey::MONOSPACE) + } + + pub fn bold_italic(&self) -> &ahash::HashSet { + self.get(FontKey::BOLD.union(FontKey::ITALIC)) + } + + pub fn bold_monospace(&self) -> &ahash::HashSet { + self.get(FontKey::BOLD.union(FontKey::MONOSPACE)) + } + + pub fn italic_monospace(&self) -> &ahash::HashSet { + self.get(FontKey::ITALIC.union(FontKey::MONOSPACE)) + } + + pub fn bold_italic_monospace(&self) -> &ahash::HashSet { + self.get(FontKey::all()) + } +} +impl std::fmt::Debug for CharacterSets { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let keys = [ + (FontKey::empty(), "regular"), + (FontKey::BOLD, "bold"), + (FontKey::ITALIC, "italic"), + (FontKey::MONOSPACE, "monospace"), + (FontKey::BOLD.union(FontKey::ITALIC), "bold_italic"), + (FontKey::BOLD.union(FontKey::MONOSPACE), "bold_monospace"), + ( + FontKey::ITALIC.union(FontKey::MONOSPACE), + "italic_monospace", + ), + (FontKey::all(), "bold_italic_monospace"), + ]; + let mut debug = f.debug_struct("CharacterSets"); + let mut had_empty = false; + for (key, name) in keys { + let chars = self.get(key); + if chars.is_empty() { + had_empty = true; + } else { + debug.field(name, chars); + } + } + if had_empty { + debug.finish_non_exhaustive() + } else { + debug.finish() + } + } +} + +bitflags! { + #[derive(Clone, Copy)] + pub struct FontKey: u8 { + const BOLD = 1; + const ITALIC = 1 << 1; + const MONOSPACE = 1 << 2; + } +} + +const FONT_VARIATIONS: usize = FontKey::all().bits() as usize + 1; + +// N.B.: html5ever 0.28.0 changed TokenSink::process_token to take &self. +// At which point we upgrade, the state on this type will need to use something +// else to provide interior mutability. +#[derive(Default)] +struct CharacterSetAccumulator { + characters: CharacterSets, + depths: [usize; 3], +} +impl CharacterSetAccumulator { + fn handle_token(&mut self, token: Token) { + match token { + Token::TagToken(tag) => { + let depth = if tag.name == local_name!("strong") || tag.name == local_name!("b") { + &mut self.depths[0] + } else if tag.name == local_name!("em") + || tag.name == local_name!("i") + || tag.attrs.iter().any(Self::is_hl_cmt) + { + &mut self.depths[1] + } else if tag.name == local_name!("code") { + &mut self.depths[2] + } else { + return; + }; + if tag.kind == TagKind::StartTag { + *depth += 1; + } else { + *depth -= 1; + } + } + Token::CharacterTokens(s) => { + let mut key = FontKey::empty(); + key.set(FontKey::BOLD, self.depths[0] > 0); + key.set(FontKey::ITALIC, self.depths[1] > 0); + key.set(FontKey::MONOSPACE, self.depths[2] > 0); + let set = self.characters.get_mut(key); + set.extend(s.chars()); + } + _ => (), + } + } + + fn is_hl_cmt(attr: &Attribute) -> bool { + attr.name.prefix == None + && attr.name.local == local_name!("class") + // this is a bit of a kludge for performance; the hl-cmt class is only + // ever used by itself, so we don't try to parse the attr value + && attr.value == "hl-cmt".into() + } +} +impl TokenSink for CharacterSetAccumulator { + type Handle = (); + fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult { + self.handle_token(token); + TokenSinkResult::Continue + } +} + +#[cfg(test)] +mod tests { + use super::get_character_sets; + + macro_rules! char_set { + ($s:ident) => { + stringify!($s).chars().collect::>() + }; + } + + #[test] + fn simple() { + let result = get_character_sets("

test

"); + assert!(result.bold().is_empty()); + assert!(result.italic().is_empty()); + assert!(result.monospace().is_empty()); + assert!(result.regular() == &char_set!(tes)); + } + + #[test] + fn formatting() { + let result = get_character_sets("

test

"); + assert!(result.bold() == &char_set!(st)); + assert!(result.italic().is_empty()); + assert!(result.monospace().is_empty()); + assert!(result.regular() == &char_set!(te)); + } + + #[test] + fn combined_formatting() { + let result = get_character_sets("ab"); + assert!(result.bold() == &char_set!(a)); + assert!(result.bold_italic() == &char_set!(b)); + } + + #[test] + fn redundant_nesting() { + let result = get_character_sets("xy"); + assert!(result.bold() == &char_set!(xy)); + } + + #[test] + fn hl_cmt_is_italic() { + let result = get_character_sets(r#"a"#); + assert!(result.italic() == &char_set!(a)); + } +} diff --git a/src/generator/css/font_subset.rs b/src/generator/css/font_subset.rs new file mode 100644 index 0000000..37ac8ee --- /dev/null +++ b/src/generator/css/font_subset.rs @@ -0,0 +1,46 @@ +use compute_graph::{ + builder::GraphBuilder, + input::{Input, InputVisitable}, + rule::Rule, + synchronicity::Asynchronous, +}; + +use super::character_sets::{CharacterSets, FontKey}; + +pub fn make_graph( + builder: &mut GraphBuilder<(), Asynchronous>, + font: Input>, + character_sets: Input, + key: FontKey, +) -> Input> { + let characters = builder.add_rule(GetCharacterSet { + sets: character_sets, + key, + }); + builder.add_rule(SubsetFont { font, characters }) +} + +#[derive(InputVisitable)] +struct GetCharacterSet { + sets: Input, + #[skip_visit] + key: FontKey, +} +impl Rule for GetCharacterSet { + type Output = ahash::HashSet; + fn evaluate(&mut self) -> Self::Output { + self.sets().get(self.key).clone() + } +} + +#[derive(InputVisitable)] +struct SubsetFont { + font: Input>, + characters: Input>, +} +impl Rule for SubsetFont { + type Output = Vec; + fn evaluate(&mut self) -> Self::Output { + self.font().clone() + } +} diff --git a/src/generator/home.rs b/src/generator/home.rs index 0f2f897..d833466 100644 --- a/src/generator/home.rs +++ b/src/generator/home.rs @@ -21,7 +21,7 @@ pub fn make_graph( posts: Input>>, default_template: Input, watcher: &mut FileWatcher, -) -> Input<()> { +) -> Input { let latest_post = builder.add_rule(LatestPost(posts)); let template_path = content_path("index.html"); diff --git a/src/generator/mod.rs b/src/generator/mod.rs index bb0fa45..e76ad7a 100644 --- a/src/generator/mod.rs +++ b/src/generator/mod.rs @@ -15,6 +15,7 @@ mod util; use std::cell::RefCell; use std::rc::Rc; +use compute_graph::input::{DynamicInput, Input}; use compute_graph::{AsyncGraph, builder::GraphBuilder}; use util::Combine; @@ -34,64 +35,71 @@ pub async fn generate(watcher: Rc>) -> anyhow::Result>) -> anyhow::Result> { let mut builder = GraphBuilder::new_async(); + let mut render_inputs: Vec> = vec![]; + let mut render_dynamic_inputs: Vec> = vec![]; + let default_template = templates::make_graph(&mut builder, &mut *watcher.borrow_mut()); - let (void_outputs, posts, all_posts) = + let (render_posts, posts, all_posts) = posts::make_graph(&mut builder, default_template.clone(), Rc::clone(&watcher)); + render_dynamic_inputs.push(render_posts); - let archive = archive::make_graph( + let render_archive = archive::make_graph( &mut builder, all_posts.clone(), default_template.clone(), &mut *watcher.borrow_mut(), ); + render_inputs.push(render_archive); - let tags = tags::make_graph( + let render_tags = tags::make_graph( &mut builder, posts.clone(), default_template.clone(), &mut *watcher.borrow_mut(), ); + render_dynamic_inputs.push(render_tags); - let home = home::make_graph( + let render_home = home::make_graph( &mut builder, all_posts, default_template.clone(), &mut *watcher.borrow_mut(), ); - - let css = css::make_graph(&mut builder, Rc::clone(&watcher)); + render_inputs.push(render_home); let statics = static_files::make_graph(&mut builder, Rc::clone(&watcher)); let rss = rss::make_graph(&mut builder, posts); - let tv = tv::make_graph(&mut builder, default_template.clone(), Rc::clone(&watcher)); + let (render_tv_series, render_tv) = + tv::make_graph(&mut builder, default_template.clone(), Rc::clone(&watcher)); + render_dynamic_inputs.push(render_tv_series); + render_inputs.push(render_tv); - let tutorials = tutorials::make_graph( + let (render_tutorial_indexes, render_tutorials) = tutorials::make_graph( &mut builder, default_template.clone(), &mut *watcher.borrow_mut(), ); + render_inputs.extend(render_tutorial_indexes); + render_dynamic_inputs.extend(render_tutorials); - let not_found = not_found::make_graph( + let render_not_found = not_found::make_graph( &mut builder, default_template.clone(), &mut *watcher.borrow_mut(), ); + render_inputs.push(render_not_found); - let output = Combine::make(&mut builder, &[ - void_outputs, - archive, - tags, - home, - css, - statics, - rss, - tv, - tutorials, - not_found, - ]); + let css = css::make_graph( + &mut builder, + render_inputs, + render_dynamic_inputs, + Rc::clone(&watcher), + ); + + let output = Combine::make(&mut builder, &[css, statics, rss]); builder.set_existing_output(output); Ok(builder.build()?) } diff --git a/src/generator/not_found.rs b/src/generator/not_found.rs index 215f56d..525beb4 100644 --- a/src/generator/not_found.rs +++ b/src/generator/not_found.rs @@ -14,18 +14,16 @@ pub fn make_graph( builder: &mut GraphBuilder<(), Asynchronous>, default_template: Input, watcher: &mut FileWatcher, -) -> Input<()> { +) -> Input { let path = content_path("404.html"); let (templates, invalidate) = builder.add_invalidatable_rule(AddTemplate::new("404", path.clone(), default_template)); watcher.watch(path, move || invalidate.invalidate()); - let render = builder.add_rule(RenderTemplate { + builder.add_rule(RenderTemplate { name: "404", output_path: "404.html".into(), templates, context: make_template_context(&"/404.html".into()).into(), - }); - - render + }) } diff --git a/src/generator/posts.rs b/src/generator/posts.rs index 07342ae..6ab8dc3 100644 --- a/src/generator/posts.rs +++ b/src/generator/posts.rs @@ -21,7 +21,7 @@ use tera::Context; use super::{ FileWatcher, templates::{AddTemplate, BuildTemplateContext, RenderTemplate, Templates}, - util::{MapDynamicToVoid, content_path}, + util::content_path, }; pub fn make_graph( @@ -29,7 +29,7 @@ pub fn make_graph( default_template: Input, watcher: Rc>, ) -> ( - Input<()>, + DynamicInput, DynamicInput, Input>>, ) { @@ -57,11 +57,7 @@ pub fn make_graph( let write_posts = builder.add_dynamic_rule(MakeWritePosts::new(html_posts.clone(), article_template)); - ( - builder.add_rule(MapDynamicToVoid(write_posts)), - posts, - builder.add_rule(AllPosts(html_posts)), - ) + (write_posts, posts, builder.add_rule(AllPosts(html_posts))) } #[derive(InputVisitable)] @@ -264,7 +260,7 @@ struct MakeWritePosts { permalink_factory: DynamicNodeFactory, output_path_factory: DynamicNodeFactory, build_context_factory: DynamicNodeFactory, - render_factory: DynamicNodeFactory, + render_factory: DynamicNodeFactory, } impl MakeWritePosts { fn new(posts: DynamicInput>>, templates: Input) -> Self { @@ -279,7 +275,7 @@ impl MakeWritePosts { } } impl DynamicRule for MakeWritePosts { - type ChildOutput = (); + type ChildOutput = String; fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { for post_input in self.posts.value().inputs.iter() { if let Some(post) = post_input.value().as_ref() { diff --git a/src/generator/tags.rs b/src/generator/tags.rs index f31a360..00f6d8c 100644 --- a/src/generator/tags.rs +++ b/src/generator/tags.rs @@ -16,7 +16,7 @@ use super::{ archive::{Entry, PostsYearMap}, posts::{ReadPostOutput, metadata::Tag}, templates::{AddTemplate, BuildTemplateContext, RenderTemplate, Templates}, - util::{MapDynamicToVoid, content_path}, + util::content_path, }; pub fn make_graph( @@ -24,7 +24,7 @@ pub fn make_graph( posts: DynamicInput, default_template: Input, watcher: &mut FileWatcher, -) -> Input<()> { +) -> DynamicInput { let by_tags = builder.add_dynamic_rule(MakePostsByTags::new(posts)); let template_path = content_path("layout/tag.html"); @@ -35,8 +35,7 @@ pub fn make_graph( )); watcher.watch(template_path, move || invalidate_template.invalidate()); - let write_tags = builder.add_dynamic_rule(MakeWriteTagPages::new(by_tags, tag_template)); - builder.add_rule(MapDynamicToVoid(write_tags)) + builder.add_dynamic_rule(MakeWriteTagPages::new(by_tags, tag_template)) } #[derive(InputVisitable)] @@ -170,7 +169,7 @@ struct MakeWriteTagPages { #[skip_visit] templates: Input, build_context_factory: DynamicNodeFactory, - render_factory: DynamicNodeFactory, + render_factory: DynamicNodeFactory, } impl MakeWriteTagPages { fn new(tags: DynamicInput, templates: Input) -> Self { @@ -183,7 +182,7 @@ impl MakeWriteTagPages { } } impl DynamicRule for MakeWriteTagPages { - type ChildOutput = (); + type ChildOutput = String; fn evaluate( &mut self, ctx: &mut impl compute_graph::rule::DynamicRuleContext, diff --git a/src/generator/templates.rs b/src/generator/templates.rs index 1870f0f..5ff24e3 100644 --- a/src/generator/templates.rs +++ b/src/generator/templates.rs @@ -1,4 +1,5 @@ use std::{ + io::Write, path::{Path, PathBuf}, time::SystemTime, }; @@ -178,27 +179,36 @@ pub struct RenderTemplate { pub context: RenderTemplateContext, } impl Rule for RenderTemplate { - type Output = (); + type Output = String; fn evaluate(&mut self) -> Self::Output { let templates = self.templates.value(); let has_template = templates.tera.get_template_names().any(|n| n == self.name); if !has_template { error!("Missing template {:?}", self.name); - return; + return String::new(); } let path: &Path = match self.output_path { TemplateOutputPath::Constant(ref p) => p, TemplateOutputPath::Dynamic(ref input) => &input.value(), }; - let writer = output_writer(path).expect("output writer"); + let mut writer = output_writer(path).expect("output writer"); let context = match self.context { RenderTemplateContext::Constant(ref ctx) => ctx, RenderTemplateContext::Dynamic(ref input) => &input.value(), }; - let result = templates.tera.render_to(&self.name, context, writer); - if let Err(e) = result { - error!("Error rendering template to {path:?}: {e:?}"); + match templates.tera.render(&self.name, context) { + Ok(str) => { + let result = writer.write_all(str.as_bytes()); + if let Err(e) = result { + error!("Error writing template to {path:?}: {e:?}"); + } + str + } + Err(e) => { + error!("Error rendering template {:?}: {:?}", self.name, e); + return String::new(); + } } } diff --git a/src/generator/tutorials.rs b/src/generator/tutorials.rs index 10de2e2..b22f332 100644 --- a/src/generator/tutorials.rs +++ b/src/generator/tutorials.rs @@ -1,11 +1,11 @@ use chrono::{DateTime, FixedOffset}; -use compute_graph::input::InputVisitable; +use compute_graph::input::{DynamicInput, InputVisitable}; use compute_graph::rule::{DynamicNodeFactory, DynamicRule, DynamicRuleContext}; use compute_graph::{builder::GraphBuilder, input::Input, synchronicity::Asynchronous}; use serde::{Deserialize, Serialize}; use crate::generator::templates::{AddTemplate, RenderTemplate, make_template_context}; -use crate::generator::util::{Combine, MapDynamicToVoid, word_count}; +use crate::generator::util::word_count; use super::markdown; use super::util::slugify::slugify; @@ -16,7 +16,7 @@ pub fn make_graph( builder: &mut GraphBuilder<(), Asynchronous>, default_template: Input, watcher: &mut FileWatcher, -) -> Input<()> { +) -> (Vec>, Vec>) { let post_path = content_path("layout/tutorial_post.html"); let (post_template, invalidate) = builder.add_invalidatable_rule(AddTemplate::new( "tutorial_post", @@ -45,10 +45,14 @@ pub fn make_graph( read_series("forge-modding-1102", "Forge Mods for 1.10.2"), read_series("forge-modding-1112", "Forge Mods for 1.11.2"), read_series("forge-modding-112", "Forge Mods for 1.12"), - ]; + ] + .into_iter() + .flatten() + .collect::>(); let mut index_entries = vec![]; - let mut render_inputs = vec![]; + let mut render_series = vec![]; + let mut render_posts = vec![]; for series in serieses { index_entries.push(TutorialIndexEntry { @@ -73,7 +77,7 @@ pub fn make_graph( series_context.insert("entries", &entries); series_context.insert("series_name", series.name); series_context.insert("series_slug", series.slug); - render_inputs.push(builder.add_rule(RenderTemplate { + render_series.push(builder.add_rule(RenderTemplate { name: "tutorial_series", output_path: format!("tutorials/{}/index.html", series.slug).into(), templates: series_template.clone(), @@ -84,25 +88,25 @@ pub fn make_graph( series.posts, post_template.clone(), )); - render_inputs.push(builder.add_rule(MapDynamicToVoid(render_dynamic))) + render_posts.push(render_dynamic) } let mut index_context = make_template_context(&"/tutorials/".into()); index_context.insert("entries", &index_entries); - render_inputs.push(builder.add_rule(RenderTemplate { + render_series.push(builder.add_rule(RenderTemplate { name: "tutorials", output_path: "tutorials/index.html".into(), templates: index_template, context: index_context.into(), })); - Combine::make(builder, &render_inputs) + (render_series, render_posts) } -fn read_series(slug: &'static str, name: &'static str) -> TutorialSeries { +fn read_series(slug: &'static str, name: &'static str) -> Option { let mut path = content_path("tutorials"); path.push(slug); - let entries = std::fs::read_dir(path).expect("reading tutorial dir"); + let entries = std::fs::read_dir(path).ok()?; let posts = entries .into_iter() .map(move |ent| { @@ -119,7 +123,7 @@ fn read_series(slug: &'static str, name: &'static str) -> TutorialSeries { }) .collect(); - TutorialSeries { name, slug, posts } + Some(TutorialSeries { name, slug, posts }) } struct TutorialSeries { @@ -179,7 +183,7 @@ struct MakeRenderTutorials { templates: Input, #[skip_visit] evaluated: bool, - render_factory: DynamicNodeFactory, + render_factory: DynamicNodeFactory, } impl MakeRenderTutorials { fn new(posts: Vec, templates: Input) -> Self { @@ -192,7 +196,7 @@ impl MakeRenderTutorials { } } impl DynamicRule for MakeRenderTutorials { - type ChildOutput = (); + type ChildOutput = String; fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { assert!(!self.evaluated); self.evaluated = true; diff --git a/src/generator/tv.rs b/src/generator/tv.rs index 6be5814..cdcc58a 100644 --- a/src/generator/tv.rs +++ b/src/generator/tv.rs @@ -27,14 +27,13 @@ use crate::generator::{ use super::{ FileWatcher, markdown, templates::{BuildTemplateContext, RenderTemplate, Templates}, - util::{Combine, MapDynamicToVoid}, }; pub fn make_graph( builder: &mut GraphBuilder<(), Asynchronous>, default_template: Input, watcher: Rc>, -) -> Input<()> { +) -> (DynamicInput, Input) { let tv_path = content_path("tv/"); let (shows, invalidate) = builder .add_invalidatable_dynamic_rule(MakeReadShows::new(tv_path.clone(), Rc::clone(&watcher))); @@ -53,7 +52,6 @@ pub fn make_graph( .watch(show_path, move || invalidate.invalidate()); let render_shows = builder.add_dynamic_rule(MakeRenderShows::new(shows.clone(), show_template)); - let void_outputs = builder.add_rule(MapDynamicToVoid(render_shows)); let tv_path = content_path("tv.html"); let (index_template, invalidate) = @@ -78,7 +76,7 @@ pub fn make_graph( context: index_context.into(), }); - builder.add_rule(Combine(void_outputs, render_index)) + (render_shows, render_index) } #[derive(InputVisitable)] @@ -274,7 +272,7 @@ struct MakeRenderShows { #[skip_visit] templates: Input, build_context_factory: DynamicNodeFactory, - render_factory: DynamicNodeFactory, + render_factory: DynamicNodeFactory, } impl MakeRenderShows { fn new(shows: DynamicInput>, templates: Input) -> Self { @@ -287,7 +285,7 @@ impl MakeRenderShows { } } impl DynamicRule for MakeRenderShows { - type ChildOutput = (); + type ChildOutput = String; fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { for show_input in self.shows.value().inputs.iter() { if let Some(show) = show_input.value().as_ref() {