From 64332a137b8d0ec7ac3d1f0ca5fc8dffd96a69b5 Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Sat, 11 Jan 2025 23:33:13 -0500 Subject: [PATCH] Add rss feed --- Cargo.lock | 132 ++++++++++++++++ Cargo.toml | 1 + crates/compute_graph/src/lib.rs | 2 +- src/generator/markdown.rs | 25 ++- src/generator/markdown/sidenotes.rs | 9 -- src/generator/markdown/simple_footnotes.rs | 48 ++++++ src/generator/mod.rs | 6 +- src/generator/posts.rs | 59 +++++-- src/generator/posts/content.rs | 18 ++- src/generator/rss.rs | 173 +++++++++++++++++++++ 10 files changed, 443 insertions(+), 30 deletions(-) create mode 100644 src/generator/markdown/simple_footnotes.rs create mode 100644 src/generator/rss.rs diff --git a/Cargo.lock b/Cargo.lock index 79b61f7..76bff89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,19 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "atom_syndication" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec03a6e158ee0f38bfba811976ae909bc2505a4a2f4049c7e8df47df3497b119" +dependencies = [ + "chrono", + "derive_builder", + "diligent-date-parser", + "never", + "quick-xml", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -351,6 +364,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.6.0" @@ -367,6 +415,37 @@ dependencies = [ "futures-util", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "derive_test" version = "0.1.0" @@ -390,6 +469,24 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "diligent-date-parser" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ede7d79366f419921e2e2f67889c12125726692a313bffb474bd5f37a581e9" +dependencies = [ + "chrono", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_filter" version = "0.1.3" @@ -805,6 +902,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "ignore" version = "0.4.23" @@ -1031,6 +1134,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "never" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -1308,6 +1417,16 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "quick-xml" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quote" version = "1.0.38" @@ -1385,6 +1504,18 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rss" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531af70fce504d369cf42ac0a9645f5a62a8ea9265de71cfa25087e9f6080c7c" +dependencies = [ + "atom_syndication", + "derive_builder", + "never", + "quick-xml", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1960,6 +2091,7 @@ dependencies = [ "once_cell", "pulldown-cmark", "regex", + "rss", "serde", "serde_json", "tera", diff --git a/Cargo.toml b/Cargo.toml index e391984..19f0446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ notify = "7.0.0" once_cell = "1.20.2" pulldown-cmark = "0.12.2" regex = "1.11.1" +rss = { version = "2.0.11", features = ["atom"] } serde = { version = "1.0", features = ["derive"] } tera = "1.20.0" tokio = { version = "1.42.0", features = ["full"] } diff --git a/crates/compute_graph/src/lib.rs b/crates/compute_graph/src/lib.rs index 1b53976..f195e87 100644 --- a/crates/compute_graph/src/lib.rs +++ b/crates/compute_graph/src/lib.rs @@ -64,7 +64,7 @@ use synchronicity::*; // use a struct for this, not a type alias, because generic bounds of type aliases aren't enforced struct NodeGraph(StableDiGraph, (), u32>); -type NodeId = petgraph::stable_graph::NodeIndex; +pub type NodeId = petgraph::stable_graph::NodeIndex; impl NodeGraph { fn new() -> Self { diff --git a/src/generator/markdown.rs b/src/generator/markdown.rs index 138b177..7d9a537 100644 --- a/src/generator/markdown.rs +++ b/src/generator/markdown.rs @@ -1,28 +1,43 @@ mod heading_anchors; mod highlight; mod sidenotes; +mod simple_footnotes; use pulldown_cmark::{Event, Options, Parser, html}; use std::io::Write; +pub fn render_undecorated(s: &str, writer: impl Write) { + html::write_html_io(writer, parse_undecorated(s)).unwrap(); +} + pub fn render(s: &str, writer: impl Write) { html::write_html_io(writer, parse(s)).unwrap(); } -pub fn parse<'a>(s: &'a str) -> impl Iterator> { +pub fn parse_base(s: &str) -> impl Iterator> { let mut options = Options::empty(); options.insert(Options::ENABLE_TABLES); options.insert(Options::ENABLE_FOOTNOTES); options.insert(Options::ENABLE_STRIKETHROUGH); options.insert(Options::ENABLE_SMART_PUNCTUATION); let parser = Parser::new_ext(s, options); - // TODO: revisit which of these stages are necessary, remove unused (and url crate dep) - let heading_anchors = heading_anchors::new(parser); - let sidenotes = sidenotes::new(heading_anchors); - let highlight = highlight::new(sidenotes); + let highlight = highlight::new(parser); highlight } +pub fn parse(s: &str) -> impl Iterator> { + let parsed = parse_base(s); + let heading_anchors = heading_anchors::new(parsed); + let sidenotes = sidenotes::new(heading_anchors); + sidenotes +} + +pub fn parse_undecorated(s: &str) -> impl Iterator> { + let parsed = parse_base(s); + let simple_footnotes = simple_footnotes::new(parsed); + simple_footnotes +} + // #[cfg(test)] // mod tests { // fn render(s: &str) -> String { diff --git a/src/generator/markdown/sidenotes.rs b/src/generator/markdown/sidenotes.rs index d9c0da7..9b4ce2f 100644 --- a/src/generator/markdown/sidenotes.rs +++ b/src/generator/markdown/sidenotes.rs @@ -202,15 +202,6 @@ mod tests { super::AdjoinFootnoteDefinitions::new(Parser::new_ext(s, options)).collect() } - fn render_sidenote_containers(s: &'static str) -> Vec> { - let mut options = Options::empty(); - options.insert(Options::ENABLE_FOOTNOTES); - super::InsertSidenoteContainers::new(super::AdjoinFootnoteDefinitions::new( - Parser::new_ext(s, options), - )) - .collect() - } - fn render(s: &'static str) -> Vec> { let mut options = Options::empty(); options.insert(Options::ENABLE_FOOTNOTES); diff --git a/src/generator/markdown/simple_footnotes.rs b/src/generator/markdown/simple_footnotes.rs new file mode 100644 index 0000000..dd894a9 --- /dev/null +++ b/src/generator/markdown/simple_footnotes.rs @@ -0,0 +1,48 @@ +use std::collections::VecDeque; + +use pulldown_cmark::{CowStr, Event, Tag, TagEnd}; + +pub fn new<'a>(inner: impl Iterator>) -> impl Iterator> { + CollectFootnotes { + inner, + in_footnote_definition: false, + footnote_events: VecDeque::new(), + } +} + +struct CollectFootnotes<'a, I: Iterator>> { + inner: I, + in_footnote_definition: bool, + footnote_events: VecDeque>, +} + +impl<'a, I: Iterator>> Iterator for CollectFootnotes<'a, I> { + type Item = Event<'a>; + fn next(&mut self) -> Option { + match self.inner.next() { + Some(Event::Start(Tag::FootnoteDefinition(id))) => { + self.in_footnote_definition = true; + let html = format!( + r#"
{id}. "# + ); + self.footnote_events.push_back(Event::Html(html.into())); + self.next() + } + Some(Event::End(TagEnd::FootnoteDefinition)) => { + self.in_footnote_definition = false; + self.footnote_events + .push_back(Event::Html(CowStr::Borrowed("
"))); + self.next() + } + Some(e) => { + if self.in_footnote_definition { + self.footnote_events.push_back(e); + self.next() + } else { + Some(e) + } + } + None => self.footnote_events.pop_front(), + } + } +} diff --git a/src/generator/mod.rs b/src/generator/mod.rs index 6f982ad..23de345 100644 --- a/src/generator/mod.rs +++ b/src/generator/mod.rs @@ -3,6 +3,7 @@ mod css; mod home; mod markdown; mod posts; +mod rss; mod static_files; mod tags; mod templates; @@ -44,7 +45,7 @@ fn make_graph(watcher: Rc>) -> anyhow::Result>) -> anyhow::Result>) -> anyhow::Result>; +pub type ReadPostOutput = Option>; #[derive(InputVisitable)] struct ReadPost { @@ -137,7 +140,7 @@ impl Rule for ReadPost { fn evaluate(&mut self) -> Self::Output { let buffer = std::fs::read_to_string(&self.path).ok()?; match Post::new(self.path.clone(), &buffer) { - Ok(post) => Some(post.to_html()), + Ok(post) => Some(post), Err(e) => { error!("Error parsing post {e:?}"); None @@ -159,6 +162,40 @@ impl Rule for ReadPost { } } +#[derive(InputVisitable)] +struct MakeConvertToHTML { + posts: DynamicInput, + factory: DynamicNodeFactory>>, +} +impl MakeConvertToHTML { + fn new(posts: DynamicInput) -> Self { + Self { + posts, + factory: DynamicNodeFactory::new(), + } + } +} +impl DynamicRule for MakeConvertToHTML { + type ChildOutput = Option>; + fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { + for post_input in self.posts.value().inputs.iter() { + self.factory.add_node(ctx, post_input.node_id(), |ctx| { + ctx.add_rule(ConvertToHTML(post_input.clone())) + }); + } + self.factory.all_nodes(ctx) + } +} + +#[derive(InputVisitable)] +struct ConvertToHTML(Input); +impl Rule for ConvertToHTML { + type Output = Option>; + fn evaluate(&mut self) -> Self::Output { + self.input_0().clone().map(|post| post.to_html()) + } +} + // #[derive(InputVisitable)] // struct MakeExtractMetadatas { // posts: DynamicInput, @@ -208,7 +245,7 @@ impl Rule for ReadPost { #[derive(InputVisitable)] struct MakeWritePosts { - posts: DynamicInput, + posts: DynamicInput>>, article_template: Input, permalink_factory: DynamicNodeFactory, output_path_factory: DynamicNodeFactory, @@ -216,7 +253,7 @@ struct MakeWritePosts { render_factory: DynamicNodeFactory, } impl MakeWritePosts { - fn new(posts: DynamicInput, templates: Input) -> Self { + fn new(posts: DynamicInput>>, templates: Input) -> Self { Self { posts, article_template: templates, @@ -276,8 +313,8 @@ impl DynamicRule for MakeWritePosts { } #[derive(InputVisitable)] -struct PostPermalink(Input); -impl Rule for PostPermalink { +struct PostPermalink(Input>>); +impl Rule for PostPermalink { type Output = String; fn evaluate(&mut self) -> Self::Output { self.input_0().as_ref().unwrap().permalink() @@ -297,7 +334,7 @@ impl Rule for PostOutputPath { /// Flattens Vec>> into Vec> #[derive(InputVisitable)] -struct AllPosts(DynamicInput); +struct AllPosts(DynamicInput>>); impl Rule for AllPosts { type Output = Vec>; fn evaluate(&mut self) -> Self::Output { diff --git a/src/generator/posts/content.rs b/src/generator/posts/content.rs index 70a8473..212868f 100644 --- a/src/generator/posts/content.rs +++ b/src/generator/posts/content.rs @@ -43,7 +43,7 @@ impl Post { path, metadata, slug: slug.to_owned(), - word_count: None, + word_count: Some(content.word_count()), excerpt: None, content, }) @@ -51,17 +51,21 @@ impl Post { } impl Post { - pub fn to_html(self) -> Post { + pub fn map_content C2>(self, make_content: F) -> Post { Post { path: self.path, metadata: self.metadata, slug: self.slug, word_count: self.word_count, excerpt: self.excerpt, - content: self.content.to_html(), + content: make_content(self.content), } } + pub fn to_html(self) -> Post { + self.map_content(|c| c.to_html()) + } + pub fn permalink(&self) -> String { format!("/{}/{}/", self.metadata.date.year(), self.slug) } @@ -92,6 +96,14 @@ impl PostContent for AnyContent { #[derive(Debug, PartialEq, Clone)] pub struct MarkdownContent(String); +impl MarkdownContent { + pub fn to_html_undecorated(self) -> HtmlContent { + let mut buf = vec![]; + markdown::render_undecorated(&self.0, &mut buf); + HtmlContent(String::from_utf8(buf).unwrap()) + } +} + impl PostContent for MarkdownContent { fn to_html(self) -> HtmlContent { let mut buf = vec![]; diff --git a/src/generator/rss.rs b/src/generator/rss.rs new file mode 100644 index 0000000..2c8474b --- /dev/null +++ b/src/generator/rss.rs @@ -0,0 +1,173 @@ +use chrono::{DateTime, Utc}; +use compute_graph::{ + NodeId, + builder::GraphBuilder, + input::{DynamicInput, Input, InputVisitable}, + node::NodeValue, + rule::{DynamicNodeFactory, DynamicRule, DynamicRuleContext, Rule}, + synchronicity::Asynchronous, +}; +use rss::{Category, ChannelBuilder, Guid, Item, ItemBuilder}; + +use super::{ + posts::{ + ReadPostOutput, + content::{AnyContent, HtmlContent, Post}, + }, + util::output_writer, +}; + +pub fn make_graph( + builder: &mut GraphBuilder<(), Asynchronous>, + posts: DynamicInput, +) -> Input<()> { + let recents = builder.add_dynamic_rule(RecentPosts::new(posts)); + let with_rss_html = builder.add_dynamic_rule(ConvertRecentsToRSSContent::new(recents)); + builder.add_rule(RenderRSSFeed(with_rss_html)) +} + +#[derive(InputVisitable)] +struct RecentPosts { + posts: DynamicInput, + factory: DynamicNodeFactory, +} +impl RecentPosts { + fn new(posts: DynamicInput) -> Self { + Self { + posts, + factory: DynamicNodeFactory::new(), + } + } +} +impl DynamicRule for RecentPosts { + type ChildOutput = ReadPostOutput; + fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { + let mut posts = vec![]; + for post_input in self.posts().inputs.iter() { + if let Some(post) = post_input.value().as_ref() { + posts.push((post_input.clone(), post.metadata.date)); + } + } + posts.sort_by_key(|post_and_date| post_and_date.1); + posts.reverse(); + for (post_input, _) in posts.into_iter().take(10) { + self.factory.add_node(ctx, post_input.node_id(), |ctx| { + ctx.add_rule(Identity(post_input)) + }); + } + self.factory.all_nodes(ctx) + } +} + +#[derive(InputVisitable)] +struct Identity(Input); +impl Rule for Identity { + type Output = T; + fn evaluate(&mut self) -> Self::Output { + self.input_0().clone() + } +} + +#[derive(InputVisitable)] +struct ConvertRecentsToRSSContent { + posts: DynamicInput, + factory: DynamicNodeFactory>>, +} +impl ConvertRecentsToRSSContent { + fn new(posts: DynamicInput) -> Self { + Self { + posts, + factory: DynamicNodeFactory::new(), + } + } +} +impl DynamicRule for ConvertRecentsToRSSContent { + type ChildOutput = Option>; + fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { + for post_input in self.posts.value().inputs.iter() { + self.factory.add_node(ctx, post_input.node_id(), |ctx| { + ctx.add_rule(ConvertPostToRSSContent(post_input.clone())) + }); + } + self.factory.all_nodes(ctx) + } +} + +#[derive(InputVisitable)] +struct ConvertPostToRSSContent(Input); +impl Rule for ConvertPostToRSSContent { + type Output = Option>; + fn evaluate(&mut self) -> Self::Output { + self.input_0() + .as_ref() + .map(|post| post.clone().map_content(convert_html_to_rss_html)) + } +} + +fn convert_html_to_rss_html(content: AnyContent) -> HtmlContent { + match content { + AnyContent::Html(html) => html, + AnyContent::Markdown(markdown) => markdown.to_html_undecorated(), + } +} + +#[derive(InputVisitable)] +struct RenderRSSFeed(DynamicInput>>); +impl Rule for RenderRSSFeed { + type Output = (); + fn evaluate(&mut self) -> Self::Output { + let posts_ = self.input_0(); + let mut posts = posts_ + .inputs + .iter() + .flat_map(|inp| inp.value().as_ref().map(|post| (inp, post.metadata.date))) + .collect::>(); + posts.sort_by_key(|post_and_date| post_and_date.1); + posts.reverse(); + let items = posts + .into_iter() + .flat_map(|(inp, _)| inp.value().as_ref().map(to_rss_item)) + .collect::>(); + let builder = ChannelBuilder::default() + .title("Shadowfacts") + .link("https://shadowfacts.net") + .last_build_date(Some(rfc_822(Utc::now()))) + .items(items) + .build(); + let writer = output_writer("/feed.xml").expect("opening feed.xml"); + builder + .pretty_write_to(writer, b' ', 2) + .expect("writing feed.xml"); + } +} + +fn rfc_822(datetime: DateTime) -> String { + datetime.format("%a, %d %b %Y %H:%M:%S %z").to_string() +} + +fn to_rss_item(post: &Post) -> Item { + ItemBuilder::default() + .title(Some(post.metadata.title.clone())) + .link(Some(format!("https://shadowfacts.net{}", post.permalink()))) + .guid(Some(Guid { + value: format!("https://shadowfacts.net{}", post.permalink()), + permalink: true, + })) + .pub_date(Some(rfc_822(post.metadata.date.into()))) + .categories( + post.metadata + .tags + .as_ref() + .map(|tags| { + tags.iter() + .map(|t| Category { + name: t.name.clone(), + domain: None, + }) + .collect::>() + }) + .unwrap_or(vec![]), + ) + .content(Some(post.content.html().to_owned())) + .build() +}