shadowfacts/v6
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Character set extraction

Browse Source
This commit is contained in:
Shadowfacts 2025-01-14 21:23:11 -05:00
parent 0be6307a41
commit 1629a7e30c
14 changed files with 537 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
Cargo.lock generated
View File

@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"getrandom",
"once_cell",
"version_check",
"zerocopy",
@ -162,9 +163,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.6.0"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be"
[[package]]
name = "block-buffer"
@ -711,7 +712,7 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757"
dependencies = [
"bitflags 2.6.0",
"bitflags 2.7.0",
"ignore",
"walkdir",
]
@ -1038,7 +1039,7 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.6.0",
"bitflags 2.7.0",
"libc",
"redox_syscall",
]
@ -1152,7 +1153,7 @@ version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009"
dependencies = [
"bitflags 2.6.0",
"bitflags 2.7.0",
"filetime",
"fsevent-sys",
"inotify",
@ -1404,7 +1405,7 @@ version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14"
dependencies = [
"bitflags 2.6.0",
"bitflags 2.7.0",
"getopts",
"memchr",
"pulldown-cmark-escape",
@ -1472,7 +1473,7 @@ version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
dependencies = [
"bitflags 2.6.0",
"bitflags 2.7.0",
]
[[package]]
@ -1933,7 +1934,7 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697"
dependencies = [
"bitflags 2.6.0",
"bitflags 2.7.0",
"bytes",
"futures-util",
"http",
@ -2235,8 +2236,10 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
name = "v7"
version = "0.1.0"
dependencies = [
"ahash",
"anyhow",
"base64",
"bitflags 2.7.0",
"chrono",
"clap",
"compute_graph",

2
Cargo.toml
View File

@ -16,8 +16,10 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
[dependencies]
ahash = "0.8.11"
anyhow = "1.0.95"
base64 = "0.22.1"
bitflags = "2.7.0"
chrono = { version = "0.4.39", features = ["serde"] }
clap = { version = "4.5.23", features = ["cargo"] }
compute_graph = { path = "crates/compute_graph" }

2
src/generator/archive.rs
View File

@ -23,7 +23,7 @@ pub fn make_graph(
posts: Input<Vec<Post<HtmlContent>>>,
default_template: Input<Templates>,
watcher: &mut FileWatcher,
) -> Input<()> {
) -> Input<String> {
let entries = builder.add_rule(Entries(posts));
let posts_by_year = builder.add_rule(PostsByYear(entries));

92
src/generator/css.rs
View File

@ -1,3 +1,6 @@
mod character_sets;
mod font_subset;
use std::{
cell::RefCell,
collections::{HashMap, HashSet},
@ -7,10 +10,11 @@ use std::{
};
use base64::{Engine, prelude::BASE64_STANDARD};
use character_sets::{CharacterSets, FontKey};
use compute_graph::{
InvalidationSignal,
builder::GraphBuilder,
input::{Input, InputVisitable, InputVisitor},
input::{DynamicInput, Input, InputVisitable, InputVisitor},
rule::Rule,
synchronicity::Asynchronous,
};
@ -20,44 +24,67 @@ use log::error;
use super::{
FileWatcher,
util::{content_path, output_writer},
util::{Combine, content_path, output_writer},
};
pub fn make_graph(
builder: &mut GraphBuilder<(), Asynchronous>,
render_inputs: Vec<Input<String>>,
render_dynamic_inputs: Vec<DynamicInput<String>>,
watcher: Rc<RefCell<FileWatcher>>,
) -> Input<()> {
let mut watcher_ = watcher.borrow_mut();
let mut files = HashMap::<&'static str, Input<String>>::new();
let filenames = [
let mut variables = HashMap::<&'static str, Input<String>>::new();
variables.insert(
"external-link",
read_to_base64(
content_path("css/external-link.svg"),
builder,
&mut *watcher_,
),
);
let character_sets = character_sets::make_graph(builder, render_inputs, render_dynamic_inputs);
let assertion = builder.add_rule(AssertNoBoldMonospace(character_sets.clone()));
let fonts = [
(
"valkyrie-a-regular",
content_path("css/fonts/valkyrie_a_regular.woff2"),
FontKey::empty(),
),
(
"valkyrie-a-bold",
content_path("css/fonts/valkyrie_a_bold.woff2"),
FontKey::BOLD,
),
(
"valkyrie-a-italic",
content_path("css/fonts/valkyrie_a_italic.woff2"),
FontKey::ITALIC,
),
(
"valkyrie-a-bold-italic",
content_path("css/fonts/valkyrie_a_bold_italic.woff2"),
FontKey::BOLD.union(FontKey::ITALIC),
),
("external-link", content_path("css/external-link.svg")),
(
"berkeley-mono-regular",
content_path("css/fonts/BerkeleyMono-Regular.woff2"),
FontKey::MONOSPACE,
),
(
"berkeley-mono-italic",
content_path("css/fonts/BerkeleyMono-Oblique.woff2"),
FontKey::MONOSPACE.union(FontKey::ITALIC),
),
];
for (name, path) in filenames.into_iter() {
files.insert(name, read_file(path, builder, &mut *watcher_));
for (name, path, font_key) in fonts.into_iter() {
let file = read_file(path, builder, &mut *watcher_);
let subsetted = font_subset::make_graph(builder, file, character_sets.clone(), font_key);
let base64 = builder.add_rule(ConvertToBase64(subsetted));
variables.insert(name, base64);
}
drop(watcher_);
@ -66,31 +93,40 @@ pub fn make_graph(
watcher,
watched: HashSet::new(),
invalidate: Rc::clone(&invalidate_css_box),
fonts: files,
variables,
});
invalidate_css_box.replace(Some(invalidate_css));
css
builder.add_rule(Combine(css, assertion))
}
fn read_file(
path: PathBuf,
builder: &mut GraphBuilder<(), Asynchronous>,
watcher: &mut FileWatcher,
) -> Input<String> {
let (font, invalidate) = builder.add_invalidatable_rule(ReadFile(path.clone()));
) -> Input<Vec<u8>> {
let (file, invalidate) = builder.add_invalidatable_rule(ReadFile(path.clone()));
watcher.watch(path, move || invalidate.invalidate());
font
file
}
fn read_to_base64(
path: PathBuf,
builder: &mut GraphBuilder<(), Asynchronous>,
watcher: &mut FileWatcher,
) -> Input<String> {
let file = read_file(path, builder, watcher);
builder.add_rule(ConvertToBase64(file))
}
struct CompileScss {
watcher: Rc<RefCell<FileWatcher>>,
watched: HashSet<PathBuf>,
invalidate: Rc<RefCell<Option<InvalidationSignal>>>,
fonts: HashMap<&'static str, Input<String>>,
variables: HashMap<&'static str, Input<String>>,
}
impl InputVisitable for CompileScss {
fn visit_inputs(&self, visitor: &mut impl InputVisitor) {
for input in self.fonts.values() {
for input in self.variables.values() {
visitor.visit(input);
}
}
@ -106,7 +142,7 @@ impl Rule for CompileScss {
OutputStyle::Compressed
};
let mut options = Options::default().fs(&fs).style(style);
for (name, input) in self.fonts.iter() {
for (name, input) in self.variables.iter() {
let value = Value::String(input.value().to_owned(), QuoteKind::None);
options = options.add_custom_var(*name, value);
}
@ -160,14 +196,14 @@ impl<'a> Fs for TrackingFs<'a> {
#[derive(InputVisitable)]
struct ReadFile(#[skip_visit] PathBuf);
impl Rule for ReadFile {
type Output = String;
type Output = Vec<u8>;
fn evaluate(&mut self) -> Self::Output {
match std::fs::read(&self.0) {
Ok(data) => BASE64_STANDARD.encode(data),
Ok(data) => data,
Err(e) => {
error!("Error reading font {:?}: {:?}", &self.0, e);
String::new()
vec![]
}
}
}
@ -176,3 +212,23 @@ impl Rule for ReadFile {
write!(f, "{}", self.0.display())
}
}
#[derive(InputVisitable)]
struct ConvertToBase64(Input<Vec<u8>>);
impl Rule for ConvertToBase64 {
type Output = String;
fn evaluate(&mut self) -> Self::Output {
BASE64_STANDARD.encode(self.input_0().as_slice())
}
}
#[derive(InputVisitable)]
struct AssertNoBoldMonospace(Input<CharacterSets>);
impl Rule for AssertNoBoldMonospace {
type Output = ();
fn evaluate(&mut self) -> Self::Output {
let sets = self.input_0();
assert!(sets.bold_monospace().is_empty());
assert!(sets.bold_italic_monospace().is_empty());
}
}

322
src/generator/css/character_sets.rs Normal file
View File

@ -0,0 +1,322 @@
use bitflags::bitflags;
use compute_graph::{
NodeId,
builder::GraphBuilder,
input::{DynamicInput, Input, InputVisitable},
rule::{DynamicNodeFactory, DynamicRule, DynamicRuleContext, Rule},
synchronicity::Asynchronous,
};
use html5ever::{
Attribute, local_name,
tokenizer::{
BufferQueue, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
TokenizerResult,
},
};
pub fn make_graph(
builder: &mut GraphBuilder<(), Asynchronous>,
inputs: Vec<Input<String>>,
dynamic_inputs: Vec<DynamicInput<String>>,
) -> Input<CharacterSets> {
let mut charsets = inputs
.into_iter()
.map(|inp| builder.add_rule(BuildCharacterSet(inp)))
.collect::<Vec<_>>();
let dynamic_charsets =
builder.add_dynamic_rule(MakeBuildDynamicCharacterSets::new(dynamic_inputs));
let dynamic_unioned = builder.add_rule(UnionDynamicCharacterSets(dynamic_charsets));
charsets.push(dynamic_unioned);
builder.add_rule(UnionCharacterSets(charsets))
}
#[derive(InputVisitable)]
struct BuildCharacterSet(Input<String>);
impl Rule for BuildCharacterSet {
type Output = CharacterSets;
fn evaluate(&mut self) -> Self::Output {
get_character_sets(&*self.input_0())
}
}
struct MakeBuildDynamicCharacterSets {
strings: Vec<DynamicInput<String>>,
factory: DynamicNodeFactory<NodeId, CharacterSets>,
}
impl MakeBuildDynamicCharacterSets {
fn new(strings: Vec<DynamicInput<String>>) -> Self {
Self {
strings,
factory: DynamicNodeFactory::new(),
}
}
}
impl InputVisitable for MakeBuildDynamicCharacterSets {
fn visit_inputs(&self, visitor: &mut impl compute_graph::input::InputVisitor) {
for input in self.strings.iter() {
input.visit_inputs(visitor);
}
}
}
impl DynamicRule for MakeBuildDynamicCharacterSets {
type ChildOutput = CharacterSets;
fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec<Input<Self::ChildOutput>> {
for dynamic_input in self.strings.iter() {
for input in dynamic_input.value().inputs.iter() {
self.factory.add_node(ctx, input.node_id(), |ctx| {
ctx.add_rule(BuildCharacterSet(input.clone()))
});
}
}
self.factory.all_nodes(ctx)
}
}
struct UnionCharacterSets(Vec<Input<CharacterSets>>);
impl InputVisitable for UnionCharacterSets {
fn visit_inputs(&self, visitor: &mut impl compute_graph::input::InputVisitor) {
for input in self.0.iter() {
input.visit_inputs(visitor);
}
}
}
impl Rule for UnionCharacterSets {
type Output = CharacterSets;
fn evaluate(&mut self) -> Self::Output {
let mut merged = CharacterSets::default();
for input in self.0.iter() {
merged.extend(&*input.value());
}
merged
}
}
#[derive(InputVisitable)]
struct UnionDynamicCharacterSets(DynamicInput<CharacterSets>);
impl Rule for UnionDynamicCharacterSets {
type Output = CharacterSets;
fn evaluate(&mut self) -> Self::Output {
let mut merged = CharacterSets::default();
for input in self.0.value().inputs.iter() {
merged.extend(&*input.value());
}
merged
}
}
fn get_character_sets(html: &str) -> CharacterSets {
let accumulator = CharacterSetAccumulator::default();
let mut tokenizer = Tokenizer::new(accumulator, TokenizerOpts::default());
let mut queue = BufferQueue::default();
queue.push_back(html.into());
let result = tokenizer.feed(&mut queue);
match result {
TokenizerResult::Done => (),
result => panic!("unexpected result: {result:?}"),
}
tokenizer.sink.characters
}
#[derive(Default, PartialEq, Clone)]
pub struct CharacterSets {
sets: [ahash::HashSet<char>; FONT_VARIATIONS],
}
impl CharacterSets {
pub fn get(&self, key: FontKey) -> &ahash::HashSet<char> {
&self.sets[key.bits() as usize]
}
pub fn get_mut(&mut self, key: FontKey) -> &mut ahash::HashSet<char> {
&mut self.sets[key.bits() as usize]
}
fn extend(&mut self, other: &CharacterSets) {
for i in 0..FONT_VARIATIONS {
self.sets[i].extend(&other.sets[i]);
}
}
}
#[allow(unused)]
impl CharacterSets {
pub fn regular(&self) -> &ahash::HashSet<char> {
&self.sets[0]
}
pub fn bold(&self) -> &ahash::HashSet<char> {
self.get(FontKey::BOLD)
}
pub fn italic(&self) -> &ahash::HashSet<char> {
self.get(FontKey::ITALIC)
}
pub fn monospace(&self) -> &ahash::HashSet<char> {
self.get(FontKey::MONOSPACE)
}
pub fn bold_italic(&self) -> &ahash::HashSet<char> {
self.get(FontKey::BOLD.union(FontKey::ITALIC))
}
pub fn bold_monospace(&self) -> &ahash::HashSet<char> {
self.get(FontKey::BOLD.union(FontKey::MONOSPACE))
}
pub fn italic_monospace(&self) -> &ahash::HashSet<char> {
self.get(FontKey::ITALIC.union(FontKey::MONOSPACE))
}
pub fn bold_italic_monospace(&self) -> &ahash::HashSet<char> {
self.get(FontKey::all())
}
}
impl std::fmt::Debug for CharacterSets {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let keys = [
(FontKey::empty(), "regular"),
(FontKey::BOLD, "bold"),
(FontKey::ITALIC, "italic"),
(FontKey::MONOSPACE, "monospace"),
(FontKey::BOLD.union(FontKey::ITALIC), "bold_italic"),
(FontKey::BOLD.union(FontKey::MONOSPACE), "bold_monospace"),
(
FontKey::ITALIC.union(FontKey::MONOSPACE),
"italic_monospace",
),
(FontKey::all(), "bold_italic_monospace"),
];
let mut debug = f.debug_struct("CharacterSets");
let mut had_empty = false;
for (key, name) in keys {
let chars = self.get(key);
if chars.is_empty() {
had_empty = true;
} else {
debug.field(name, chars);
}
}
if had_empty {
debug.finish_non_exhaustive()
} else {
debug.finish()
}
}
}
bitflags! {
#[derive(Clone, Copy)]
pub struct FontKey: u8 {
const BOLD = 1;
const ITALIC = 1 << 1;
const MONOSPACE = 1 << 2;
}
}
const FONT_VARIATIONS: usize = FontKey::all().bits() as usize + 1;
// N.B.: html5ever 0.28.0 changed TokenSink::process_token to take &self.
// At which point we upgrade, the state on this type will need to use something
// else to provide interior mutability.
#[derive(Default)]
struct CharacterSetAccumulator {
characters: CharacterSets,
depths: [usize; 3],
}
impl CharacterSetAccumulator {
fn handle_token(&mut self, token: Token) {
match token {
Token::TagToken(tag) => {
let depth = if tag.name == local_name!("strong") || tag.name == local_name!("b") {
&mut self.depths[0]
} else if tag.name == local_name!("em")
|| tag.name == local_name!("i")
|| tag.attrs.iter().any(Self::is_hl_cmt)
{
&mut self.depths[1]
} else if tag.name == local_name!("code") {
&mut self.depths[2]
} else {
return;
};
if tag.kind == TagKind::StartTag {
*depth += 1;
} else {
*depth -= 1;
}
}
Token::CharacterTokens(s) => {
let mut key = FontKey::empty();
key.set(FontKey::BOLD, self.depths[0] > 0);
key.set(FontKey::ITALIC, self.depths[1] > 0);
key.set(FontKey::MONOSPACE, self.depths[2] > 0);
let set = self.characters.get_mut(key);
set.extend(s.chars());
}
_ => (),
}
}
fn is_hl_cmt(attr: &Attribute) -> bool {
attr.name.prefix == None
&& attr.name.local == local_name!("class")
// this is a bit of a kludge for performance; the hl-cmt class is only
// ever used by itself, so we don't try to parse the attr value
&& attr.value == "hl-cmt".into()
}
}
impl TokenSink for CharacterSetAccumulator {
type Handle = ();
fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult<Self::Handle> {
self.handle_token(token);
TokenSinkResult::Continue
}
}
#[cfg(test)]
mod tests {
use super::get_character_sets;
macro_rules! char_set {
($s:ident) => {
stringify!($s).chars().collect::<ahash::HashSet<char>>()
};
}
#[test]
fn simple() {
let result = get_character_sets("<p>test</p>");
assert!(result.bold().is_empty());
assert!(result.italic().is_empty());
assert!(result.monospace().is_empty());
assert!(result.regular() == &char_set!(tes));
}
#[test]
fn formatting() {
let result = get_character_sets("<p>te<strong>st</strong></p>");
assert!(result.bold() == &char_set!(st));
assert!(result.italic().is_empty());
assert!(result.monospace().is_empty());
assert!(result.regular() == &char_set!(te));
}
#[test]
fn combined_formatting() {
let result = get_character_sets("<strong>a<em>b</em></strong>");
assert!(result.bold() == &char_set!(a));
assert!(result.bold_italic() == &char_set!(b));
}
#[test]
fn redundant_nesting() {
let result = get_character_sets("<b><b>x</b>y</b>");
assert!(result.bold() == &char_set!(xy));
}
#[test]
fn hl_cmt_is_italic() {
let result = get_character_sets(r#"<span class="hl-cmt">a</span>"#);
assert!(result.italic() == &char_set!(a));
}
}

46
src/generator/css/font_subset.rs Normal file
View File

@ -0,0 +1,46 @@
use compute_graph::{
builder::GraphBuilder,
input::{Input, InputVisitable},
rule::Rule,
synchronicity::Asynchronous,
};
use super::character_sets::{CharacterSets, FontKey};
pub fn make_graph(
builder: &mut GraphBuilder<(), Asynchronous>,
font: Input<Vec<u8>>,
character_sets: Input<CharacterSets>,
key: FontKey,
) -> Input<Vec<u8>> {
let characters = builder.add_rule(GetCharacterSet {
sets: character_sets,
key,
});
builder.add_rule(SubsetFont { font, characters })
}
#[derive(InputVisitable)]
struct GetCharacterSet {
sets: Input<CharacterSets>,
#[skip_visit]
key: FontKey,
}
impl Rule for GetCharacterSet {
type Output = ahash::HashSet<char>;
fn evaluate(&mut self) -> Self::Output {
self.sets().get(self.key).clone()
}
}
#[derive(InputVisitable)]
struct SubsetFont {
font: Input<Vec<u8>>,
characters: Input<ahash::HashSet<char>>,
}
impl Rule for SubsetFont {
type Output = Vec<u8>;
fn evaluate(&mut self) -> Self::Output {
self.font().clone()
}
}

2
src/generator/home.rs
View File

@ -21,7 +21,7 @@ pub fn make_graph(
posts: Input<Vec<Post<HtmlContent>>>,
default_template: Input<Templates>,
watcher: &mut FileWatcher,
) -> Input<()> {
) -> Input<String> {
let latest_post = builder.add_rule(LatestPost(posts));
let template_path = content_path("index.html");

50
src/generator/mod.rs
View File

@ -15,6 +15,7 @@ mod util;
use std::cell::RefCell;
use std::rc::Rc;
use compute_graph::input::{DynamicInput, Input};
use compute_graph::{AsyncGraph, builder::GraphBuilder};
use util::Combine;
@ -34,64 +35,71 @@ pub async fn generate(watcher: Rc<RefCell<FileWatcher>>) -> anyhow::Result<Async
fn make_graph(watcher: Rc<RefCell<FileWatcher>>) -> anyhow::Result<AsyncGraph<()>> {
let mut builder = GraphBuilder::new_async();
let mut render_inputs: Vec<Input<String>> = vec![];
let mut render_dynamic_inputs: Vec<DynamicInput<String>> = vec![];
let default_template = templates::make_graph(&mut builder, &mut *watcher.borrow_mut());
let (void_outputs, posts, all_posts) =
let (render_posts, posts, all_posts) =
posts::make_graph(&mut builder, default_template.clone(), Rc::clone(&watcher));
render_dynamic_inputs.push(render_posts);
let archive = archive::make_graph(
let render_archive = archive::make_graph(
&mut builder,
all_posts.clone(),
default_template.clone(),
&mut *watcher.borrow_mut(),
);
render_inputs.push(render_archive);
let tags = tags::make_graph(
let render_tags = tags::make_graph(
&mut builder,
posts.clone(),
default_template.clone(),
&mut *watcher.borrow_mut(),
);
render_dynamic_inputs.push(render_tags);
let home = home::make_graph(
let render_home = home::make_graph(
&mut builder,
all_posts,
default_template.clone(),
&mut *watcher.borrow_mut(),
);
let css = css::make_graph(&mut builder, Rc::clone(&watcher));
render_inputs.push(render_home);
let statics = static_files::make_graph(&mut builder, Rc::clone(&watcher));
let rss = rss::make_graph(&mut builder, posts);
let tv = tv::make_graph(&mut builder, default_template.clone(), Rc::clone(&watcher));
let (render_tv_series, render_tv) =
tv::make_graph(&mut builder, default_template.clone(), Rc::clone(&watcher));
render_dynamic_inputs.push(render_tv_series);
render_inputs.push(render_tv);
let tutorials = tutorials::make_graph(
let (render_tutorial_indexes, render_tutorials) = tutorials::make_graph(
&mut builder,
default_template.clone(),
&mut *watcher.borrow_mut(),
);
render_inputs.extend(render_tutorial_indexes);
render_dynamic_inputs.extend(render_tutorials);
let not_found = not_found::make_graph(
let render_not_found = not_found::make_graph(
&mut builder,
default_template.clone(),
&mut *watcher.borrow_mut(),
);
render_inputs.push(render_not_found);
let output = Combine::make(&mut builder, &[
void_outputs,
archive,
tags,
home,
css,
statics,
rss,
tv,
tutorials,
not_found,
]);
let css = css::make_graph(
&mut builder,
render_inputs,
render_dynamic_inputs,
Rc::clone(&watcher),
);
let output = Combine::make(&mut builder, &[css, statics, rss]);
builder.set_existing_output(output);
Ok(builder.build()?)
}

8
src/generator/not_found.rs
View File

@ -14,18 +14,16 @@ pub fn make_graph(
builder: &mut GraphBuilder<(), Asynchronous>,
default_template: Input<Templates>,
watcher: &mut FileWatcher,
) -> Input<()> {
) -> Input<String> {
let path = content_path("404.html");
let (templates, invalidate) =
builder.add_invalidatable_rule(AddTemplate::new("404", path.clone(), default_template));
watcher.watch(path, move || invalidate.invalidate());
let render = builder.add_rule(RenderTemplate {
builder.add_rule(RenderTemplate {
name: "404",
output_path: "404.html".into(),
templates,
context: make_template_context(&"/404.html".into()).into(),
});
render
})
}

14
src/generator/posts.rs
View File

@ -21,7 +21,7 @@ use tera::Context;
use super::{
FileWatcher,
templates::{AddTemplate, BuildTemplateContext, RenderTemplate, Templates},
util::{MapDynamicToVoid, content_path},
util::content_path,
};
pub fn make_graph(
@ -29,7 +29,7 @@ pub fn make_graph(
default_template: Input<Templates>,
watcher: Rc<RefCell<FileWatcher>>,
) -> (
Input<()>,
DynamicInput<String>,
DynamicInput<ReadPostOutput>,
Input<Vec<Post<HtmlContent>>>,
) {
@ -57,11 +57,7 @@ pub fn make_graph(
let write_posts =
builder.add_dynamic_rule(MakeWritePosts::new(html_posts.clone(), article_template));
(
builder.add_rule(MapDynamicToVoid(write_posts)),
posts,
builder.add_rule(AllPosts(html_posts)),
)
(write_posts, posts, builder.add_rule(AllPosts(html_posts)))
}
#[derive(InputVisitable)]
@ -264,7 +260,7 @@ struct MakeWritePosts {
permalink_factory: DynamicNodeFactory<PathBuf, String>,
output_path_factory: DynamicNodeFactory<PathBuf, PathBuf>,
build_context_factory: DynamicNodeFactory<PathBuf, Context>,
render_factory: DynamicNodeFactory<PathBuf, ()>,
render_factory: DynamicNodeFactory<PathBuf, String>,
}
impl MakeWritePosts {
fn new(posts: DynamicInput<Option<Post<HtmlContent>>>, templates: Input<Templates>) -> Self {
@ -279,7 +275,7 @@ impl MakeWritePosts {
}
}
impl DynamicRule for MakeWritePosts {
type ChildOutput = ();
type ChildOutput = String;
fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec<Input<Self::ChildOutput>> {
for post_input in self.posts.value().inputs.iter() {
if let Some(post) = post_input.value().as_ref() {

11
src/generator/tags.rs
View File

@ -16,7 +16,7 @@ use super::{
archive::{Entry, PostsYearMap},
posts::{ReadPostOutput, metadata::Tag},
templates::{AddTemplate, BuildTemplateContext, RenderTemplate, Templates},
util::{MapDynamicToVoid, content_path},
util::content_path,
};
pub fn make_graph(
@ -24,7 +24,7 @@ pub fn make_graph(
posts: DynamicInput<ReadPostOutput>,
default_template: Input<Templates>,
watcher: &mut FileWatcher,
) -> Input<()> {
) -> DynamicInput<String> {
let by_tags = builder.add_dynamic_rule(MakePostsByTags::new(posts));
let template_path = content_path("layout/tag.html");
@ -35,8 +35,7 @@ pub fn make_graph(
));
watcher.watch(template_path, move || invalidate_template.invalidate());
let write_tags = builder.add_dynamic_rule(MakeWriteTagPages::new(by_tags, tag_template));
builder.add_rule(MapDynamicToVoid(write_tags))
builder.add_dynamic_rule(MakeWriteTagPages::new(by_tags, tag_template))
}
#[derive(InputVisitable)]
@ -170,7 +169,7 @@ struct MakeWriteTagPages {
#[skip_visit]
templates: Input<Templates>,
build_context_factory: DynamicNodeFactory<String, Context>,
render_factory: DynamicNodeFactory<String, ()>,
render_factory: DynamicNodeFactory<String, String>,
}
impl MakeWriteTagPages {
fn new(tags: DynamicInput<TagAndPosts>, templates: Input<Templates>) -> Self {
@ -183,7 +182,7 @@ impl MakeWriteTagPages {
}
}
impl DynamicRule for MakeWriteTagPages {
type ChildOutput = ();
type ChildOutput = String;
fn evaluate(
&mut self,
ctx: &mut impl compute_graph::rule::DynamicRuleContext,

20
src/generator/templates.rs
View File

@ -1,4 +1,5 @@
use std::{
io::Write,
path::{Path, PathBuf},
time::SystemTime,
};
@ -178,27 +179,36 @@ pub struct RenderTemplate {
pub context: RenderTemplateContext,
}
impl Rule for RenderTemplate {
type Output = ();
type Output = String;
fn evaluate(&mut self) -> Self::Output {
let templates = self.templates.value();
let has_template = templates.tera.get_template_names().any(|n| n == self.name);
if !has_template {
error!("Missing template {:?}", self.name);
return;
return String::new();
}
let path: &Path = match self.output_path {
TemplateOutputPath::Constant(ref p) => p,
TemplateOutputPath::Dynamic(ref input) => &input.value(),
};
let writer = output_writer(path).expect("output writer");
let mut writer = output_writer(path).expect("output writer");
let context = match self.context {
RenderTemplateContext::Constant(ref ctx) => ctx,
RenderTemplateContext::Dynamic(ref input) => &input.value(),
};
let result = templates.tera.render_to(&self.name, context, writer);
match templates.tera.render(&self.name, context) {
Ok(str) => {
let result = writer.write_all(str.as_bytes());
if let Err(e) = result {
error!("Error rendering template to {path:?}: {e:?}");
error!("Error writing template to {path:?}: {e:?}");
}
str
}
Err(e) => {
error!("Error rendering template {:?}: {:?}", self.name, e);
return String::new();
}
}
}

32
src/generator/tutorials.rs
View File

@ -1,11 +1,11 @@
use chrono::{DateTime, FixedOffset};
use compute_graph::input::InputVisitable;
use compute_graph::input::{DynamicInput, InputVisitable};
use compute_graph::rule::{DynamicNodeFactory, DynamicRule, DynamicRuleContext};
use compute_graph::{builder::GraphBuilder, input::Input, synchronicity::Asynchronous};
use serde::{Deserialize, Serialize};
use crate::generator::templates::{AddTemplate, RenderTemplate, make_template_context};
use crate::generator::util::{Combine, MapDynamicToVoid, word_count};
use crate::generator::util::word_count;
use super::markdown;
use super::util::slugify::slugify;
@ -16,7 +16,7 @@ pub fn make_graph(
builder: &mut GraphBuilder<(), Asynchronous>,
default_template: Input<Templates>,
watcher: &mut FileWatcher,
) -> Input<()> {
) -> (Vec<Input<String>>, Vec<DynamicInput<String>>) {
let post_path = content_path("layout/tutorial_post.html");
let (post_template, invalidate) = builder.add_invalidatable_rule(AddTemplate::new(
"tutorial_post",
@ -45,10 +45,14 @@ pub fn make_graph(
read_series("forge-modding-1102", "Forge Mods for 1.10.2"),
read_series("forge-modding-1112", "Forge Mods for 1.11.2"),
read_series("forge-modding-112", "Forge Mods for 1.12"),
];
]
.into_iter()
.flatten()
.collect::<Vec<_>>();
let mut index_entries = vec![];
let mut render_inputs = vec![];
let mut render_series = vec![];
let mut render_posts = vec![];
for series in serieses {
index_entries.push(TutorialIndexEntry {
@ -73,7 +77,7 @@ pub fn make_graph(
series_context.insert("entries", &entries);
series_context.insert("series_name", series.name);
series_context.insert("series_slug", series.slug);
render_inputs.push(builder.add_rule(RenderTemplate {
render_series.push(builder.add_rule(RenderTemplate {
name: "tutorial_series",
output_path: format!("tutorials/{}/index.html", series.slug).into(),
templates: series_template.clone(),
@ -84,25 +88,25 @@ pub fn make_graph(
series.posts,
post_template.clone(),
));
render_inputs.push(builder.add_rule(MapDynamicToVoid(render_dynamic)))
render_posts.push(render_dynamic)
}
let mut index_context = make_template_context(&"/tutorials/".into());
index_context.insert("entries", &index_entries);
render_inputs.push(builder.add_rule(RenderTemplate {
render_series.push(builder.add_rule(RenderTemplate {
name: "tutorials",
output_path: "tutorials/index.html".into(),
templates: index_template,
context: index_context.into(),
}));
Combine::make(builder, &render_inputs)
(render_series, render_posts)
}
fn read_series(slug: &'static str, name: &'static str) -> TutorialSeries {
fn read_series(slug: &'static str, name: &'static str) -> Option<TutorialSeries> {
let mut path = content_path("tutorials");
path.push(slug);
let entries = std::fs::read_dir(path).expect("reading tutorial dir");
let entries = std::fs::read_dir(path).ok()?;
let posts = entries
.into_iter()
.map(move |ent| {
@ -119,7 +123,7 @@ fn read_series(slug: &'static str, name: &'static str) -> TutorialSeries {
})
.collect();
TutorialSeries { name, slug, posts }
Some(TutorialSeries { name, slug, posts })
}
struct TutorialSeries {
@ -179,7 +183,7 @@ struct MakeRenderTutorials {
templates: Input<Templates>,
#[skip_visit]
evaluated: bool,
render_factory: DynamicNodeFactory<String, ()>,
render_factory: DynamicNodeFactory<String, String>,
}
impl MakeRenderTutorials {
fn new(posts: Vec<TutorialPost>, templates: Input<Templates>) -> Self {
@ -192,7 +196,7 @@ impl MakeRenderTutorials {
}
}
impl DynamicRule for MakeRenderTutorials {
type ChildOutput = ();
type ChildOutput = String;
fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec<Input<Self::ChildOutput>> {
assert!(!self.evaluated);
self.evaluated = true;

10
src/generator/tv.rs
View File

@ -27,14 +27,13 @@ use crate::generator::{
use super::{
FileWatcher, markdown,
templates::{BuildTemplateContext, RenderTemplate, Templates},
util::{Combine, MapDynamicToVoid},
};
pub fn make_graph(
builder: &mut GraphBuilder<(), Asynchronous>,
default_template: Input<Templates>,
watcher: Rc<RefCell<FileWatcher>>,
) -> Input<()> {
) -> (DynamicInput<String>, Input<String>) {
let tv_path = content_path("tv/");
let (shows, invalidate) = builder
.add_invalidatable_dynamic_rule(MakeReadShows::new(tv_path.clone(), Rc::clone(&watcher)));
@ -53,7 +52,6 @@ pub fn make_graph(
.watch(show_path, move || invalidate.invalidate());
let render_shows = builder.add_dynamic_rule(MakeRenderShows::new(shows.clone(), show_template));
let void_outputs = builder.add_rule(MapDynamicToVoid(render_shows));
let tv_path = content_path("tv.html");
let (index_template, invalidate) =
@ -78,7 +76,7 @@ pub fn make_graph(
context: index_context.into(),
});
builder.add_rule(Combine(void_outputs, render_index))
(render_shows, render_index)
}
#[derive(InputVisitable)]
@ -274,7 +272,7 @@ struct MakeRenderShows {
#[skip_visit]
templates: Input<Templates>,
build_context_factory: DynamicNodeFactory<String, Context>,
render_factory: DynamicNodeFactory<String, ()>,
render_factory: DynamicNodeFactory<String, String>,
}
impl MakeRenderShows {
fn new(shows: DynamicInput<Option<Show>>, templates: Input<Templates>) -> Self {
@ -287,7 +285,7 @@ impl MakeRenderShows {
}
}
impl DynamicRule for MakeRenderShows {
type ChildOutput = ();
type ChildOutput = String;
fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec<Input<Self::ChildOutput>> {
for show_input in self.shows.value().inputs.iter() {
if let Some(show) = show_input.value().as_ref() {