From 2c1b9c620e852b96b6e6e41ccbcf7b24bdc7abfa Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Tue, 31 Dec 2024 18:58:47 -0500 Subject: [PATCH] File watching --- Cargo.lock | 283 ++++++++++++++++++++++++++++- Cargo.toml | 4 + src/generator/mod.rs | 18 +- src/generator/posts.rs | 27 ++- src/generator/util/file_watcher.rs | 80 ++++++++ src/generator/util/mod.rs | 7 +- src/main.rs | 57 +++++- 7 files changed, 457 insertions(+), 19 deletions(-) create mode 100644 src/generator/util/file_watcher.rs diff --git a/Cargo.lock b/Cargo.lock index 33509dc..50fba3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,6 +170,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.6.0" @@ -283,6 +289,16 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "debounced" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "107e5cd9b5163c19751e53eef634cae25cf5ed5f6d0c81125feaa92e43703cc7" +dependencies = [ + "futures-timer", + "futures-util", +] + [[package]] name = "derive_test" version = "0.1.0" @@ -330,6 +346,18 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -345,6 +373,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futf" version = "0.1.5" @@ -355,6 +392,101 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getopts" version = "0.2.21" @@ -588,6 +720,35 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -610,6 +771,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "libc" version = "0.2.169" @@ -622,6 +803,17 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall", +] + [[package]] name = "litemap" version = "0.7.4" @@ -720,6 +912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.52.0", ] @@ -740,6 +933,34 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "notify" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +dependencies = [ + "bitflags 2.6.0", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.52.0", +] + +[[package]] +name = "notify-types" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585d3cb5e12e01aed9e8a1f70d5c6b5e86fe2a6e48fc8cd0b3e0b8df6f6eb174" +dependencies = [ + "instant", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -866,6 +1087,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -896,7 +1123,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags", + "bitflags 2.6.0", "getopts", "memchr", "pulldown-cmark-escape", @@ -954,7 +1181,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags", + "bitflags 2.6.0", ] [[package]] @@ -998,6 +1225,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1066,6 +1302,15 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -1207,6 +1452,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.19" @@ -1312,21 +1568,35 @@ dependencies = [ "chrono", "clap", "compute_graph", + "debounced", "env_logger", + "futures", "html5ever", "log", "markup5ever_rcdom", + "notify", "once_cell", "pulldown-cmark", "regex", "serde", "serde_json", "tokio", + "tokio-stream", "toml", "unicode-normalization", "url", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1387,6 +1657,15 @@ version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "windows-core" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index d7f43f1..dfa420e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,15 +21,19 @@ askama = "0.12.1" chrono = { version = "0.4.39", features = ["serde"] } clap = { version = "4.5.23", features = ["cargo"] } compute_graph = { path = "crates/compute_graph" } +debounced = "0.2.0" env_logger = "0.11.6" +futures = "0.3.31" html5ever = "0.27.0" log = "0.4.22" markup5ever_rcdom = "0.3.0" +notify = "7.0.0" once_cell = "1.20.2" pulldown-cmark = "0.12.2" regex = "1.11.1" serde = { version = "1.0", features = ["derive"] } tokio = { version = "1.42.0", features = ["full"] } +tokio-stream = "0.1.17" toml = "0.8.19" unicode-normalization = "0.1.24" url = "2.5.4" diff --git a/src/generator/mod.rs b/src/generator/mod.rs index a0781f1..15aa363 100644 --- a/src/generator/mod.rs +++ b/src/generator/mod.rs @@ -4,27 +4,31 @@ mod posts; mod tags; mod util; +use std::cell::RefCell; +use std::rc::Rc; + use compute_graph::{AsyncGraph, builder::GraphBuilder}; use util::{Combine, MapToVoid}; -pub async fn generate() -> anyhow::Result<()> { +pub use util::content_base_path; +pub use util::file_watcher::FileWatcher; + +pub async fn generate(watcher: Rc>) -> anyhow::Result> { std::fs::create_dir_all("out").expect("creating output dir"); - // TODO: file watching - - let mut graph = make_graph()?; + let mut graph = make_graph(watcher)?; graph.evaluate_async().await; println!("{}", graph.as_dot_string()); - Ok(()) + Ok(graph) } -fn make_graph() -> anyhow::Result> { +fn make_graph(watcher: Rc>) -> anyhow::Result> { let mut builder = GraphBuilder::new_async(); - let (void_outputs, posts, all_posts, post_metadatas) = posts::make_graph(&mut builder); + let (void_outputs, posts, all_posts, post_metadatas) = posts::make_graph(&mut builder, watcher); let archive = archive::make_graph(&mut builder, all_posts); diff --git a/src/generator/posts.rs b/src/generator/posts.rs index 7be8072..99371d5 100644 --- a/src/generator/posts.rs +++ b/src/generator/posts.rs @@ -1,7 +1,9 @@ pub mod content; pub mod metadata; +use std::cell::RefCell; use std::path::PathBuf; +use std::rc::Rc; use askama::Template; use compute_graph::{ @@ -18,19 +20,26 @@ use metadata::PostMetadata; use crate::generator::util::output_rendered_template; -use super::util::{MapDynamicToVoid, content_path, templates::TemplateCommon}; +use super::{ + FileWatcher, + util::{MapDynamicToVoid, content_path, templates::TemplateCommon}, +}; pub fn make_graph( builder: &mut GraphBuilder<(), Asynchronous>, + watcher: Rc>, ) -> ( Input<()>, DynamicInput, Input>>, Input>, ) { - // todo: make this invalidatable, watch files - let post_files = builder.add_rule(ListPostFiles); - let posts = builder.add_dynamic_rule(MakeReadNodes::new(post_files)); + let (post_files, invalidate_posts) = builder.add_invalidatable_rule(ListPostFiles); + watcher.borrow_mut().watch(content_path("posts/"), move || { + invalidate_posts.invalidate(); + }); + + let posts = builder.add_dynamic_rule(MakeReadNodes::new(post_files, watcher)); let extract_metadatas = builder.add_dynamic_rule(MakeExtractMetadatas::new(posts.clone())); @@ -80,12 +89,14 @@ fn find_index(path: PathBuf) -> Option { #[derive(InputVisitable)] struct MakeReadNodes { files: Input>, + watcher: Rc>, node_factory: DynamicNodeFactory, } impl MakeReadNodes { - fn new(files: Input>) -> Self { + fn new(files: Input>, watcher: Rc>) -> Self { Self { files, + watcher, node_factory: DynamicNodeFactory::new(), } } @@ -95,7 +106,11 @@ impl DynamicRule for MakeReadNodes { fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec> { for file in self.files.value().iter() { self.node_factory.add_rule(ctx, file.clone(), |ctx| { - ctx.add_rule(ReadPost { path: file.clone() }) + let (input, signal) = ctx.add_invalidatable_rule(ReadPost { path: file.clone() }); + self.watcher + .borrow_mut() + .watch(file.clone(), move || signal.invalidate()); + input }); } self.node_factory.all_nodes(ctx) diff --git a/src/generator/util/file_watcher.rs b/src/generator/util/file_watcher.rs new file mode 100644 index 0000000..6549c6a --- /dev/null +++ b/src/generator/util/file_watcher.rs @@ -0,0 +1,80 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use notify::{EventHandler, Watcher}; +use tokio::sync::mpsc::UnboundedSender; + +use crate::generator::util::content_base_path; + +pub struct FileWatcher { + handlers: HashMap ()>>, + watcher: Option, +} + +impl FileWatcher { + pub fn new() -> Self { + Self { + handlers: HashMap::new(), + watcher: None, + } + } + + pub fn watch(&mut self, path: PathBuf, f: impl Fn() -> () + 'static) { + if let Some(existing) = self.handlers.remove(&path) { + self.handlers.insert( + path, + Box::new(move || { + existing(); + f(); + }), + ); + } else { + self.handlers.insert(path, Box::new(f)); + } + } + + pub async fn start(&mut self) -> anyhow::Result<()> { + assert!(self.watcher.is_none()); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::>(); + + let mut watcher = notify::recommended_watcher(AsyncEventHandler(tx))?; + watcher.watch(&content_base_path(), notify::RecursiveMode::Recursive)?; + self.watcher = Some(watcher); + + let mut absolute_content_parent = content_base_path().canonicalize()?; + absolute_content_parent.pop(); + + while let Some(result) = rx.recv().await { + if let Ok(ev) = result { + self.handle_event(ev, &absolute_content_parent); + } + } + + Ok(()) + } + + fn handle_event(&self, event: notify::Event, base: &Path) { + for path in event.paths { + let relative = path + .strip_prefix(base) + .expect("should only receive events for paths in content path"); + let mut path = PathBuf::new(); + for component in relative { + path.push(component); + if let Some(handler) = self.handlers.get(&path) { + handler(); + } + } + } + } +} + +struct AsyncEventHandler(UnboundedSender>); + +impl EventHandler for AsyncEventHandler { + fn handle_event(&mut self, event: notify::Result) { + self.0.send(event).expect("sending event"); + } +} diff --git a/src/generator/util/mod.rs b/src/generator/util/mod.rs index b7e5a40..df9ae4c 100644 --- a/src/generator/util/mod.rs +++ b/src/generator/util/mod.rs @@ -1,3 +1,4 @@ +pub mod file_watcher; pub mod one_more; pub mod slugify; pub mod templates; @@ -80,8 +81,12 @@ pub fn from_frontmatter(contents: &str) -> anyhow::Result<( Ok((deserialized, chars.as_str())) } +pub fn content_base_path() -> PathBuf { + PathBuf::from("site_test/") +} + pub fn content_path(p: impl AsRef) -> PathBuf { - let mut buf = PathBuf::from("site_test/"); + let mut buf = content_base_path(); join_abs(&mut buf, p.as_ref()); buf } diff --git a/src/main.rs b/src/main.rs index 3091f82..8ccbbd8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,16 @@ mod generator; use clap::{Command, arg, command}; +use debounced::debounced; +use futures::FutureExt; +use generator::{FileWatcher, content_base_path}; use log::info; +use std::cell::RefCell; +use std::rc::Rc; +use std::time::Duration; +use tokio::pin; +use tokio_stream::StreamExt; +use tokio_stream::wrappers::UnboundedReceiverStream; #[tokio::main] async fn main() { @@ -12,7 +21,10 @@ async fn main() { let matches = command!() .subcommand_required(true) .arg_required_else_help(true) - .subcommand(Command::new("gen")) + .subcommand( + Command::new("gen") + .arg(arg!(--watch "Watch the site directory and regenerate on changes")), + ) .subcommand( Command::new("serve") .arg(arg!(--watch "Watch the site directory and regenerate on changes")), @@ -26,8 +38,47 @@ async fn main() { } match matches.subcommand() { - Some(("gen", _)) => { - generator::generate().await.expect("generating"); + Some(("gen", matches)) => { + let watcher = Rc::new(RefCell::new(FileWatcher::new())); + + let mut graph = generator::generate(Rc::clone(&watcher)) + .await + .expect("generating"); + + if matches.contains_id("watch") { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<()>(); + + let mut watcher = watcher.borrow_mut(); + watcher.watch(content_base_path(), move || { + tx.send(()).expect("sending regenerate signal"); + }); + + let mut debounced = + debounced(UnboundedReceiverStream::new(rx), Duration::from_millis(500)); + + let watch = watcher.start().fuse(); + + let regenerate = async move { + while let Some(_) = debounced.next().await { + info!("Regenerating"); + graph.evaluate_async().await; + } + } + .fuse(); + + pin!(regenerate, watch); + + loop { + futures::select! { + watcher_res = watch => { + watcher_res.expect("watching files"); + } + _ = regenerate => { + info!("regenerate channel closed"); + } + } + } + } } Some(("serve", _matches)) => { todo!()