File watching

This commit is contained in:
Shadowfacts 2024-12-31 18:58:47 -05:00
parent 640c0ab620
commit 2c1b9c620e
7 changed files with 457 additions and 19 deletions

283
Cargo.lock generated
View File

@ -170,6 +170,12 @@ dependencies = [
"serde",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.6.0"
@ -283,6 +289,16 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "debounced"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "107e5cd9b5163c19751e53eef634cae25cf5ed5f6d0c81125feaa92e43703cc7"
dependencies = [
"futures-timer",
"futures-util",
]
[[package]]
name = "derive_test"
version = "0.1.0"
@ -330,6 +346,18 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "filetime"
version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
dependencies = [
"cfg-if",
"libc",
"libredox",
"windows-sys 0.59.0",
]
[[package]]
name = "fixedbitset"
version = "0.5.7"
@ -345,6 +373,15 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fsevent-sys"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
dependencies = [
"libc",
]
[[package]]
name = "futf"
version = "0.1.5"
@ -355,6 +392,101 @@ dependencies = [
"new_debug_unreachable",
]
[[package]]
name = "futures"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
name = "futures-core"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
[[package]]
name = "futures-executor"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-macro"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
[[package]]
name = "futures-task"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
[[package]]
name = "futures-timer"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
[[package]]
name = "futures-util"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
name = "getopts"
version = "0.2.21"
@ -588,6 +720,35 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "inotify"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc"
dependencies = [
"bitflags 1.3.2",
"inotify-sys",
"libc",
]
[[package]]
name = "inotify-sys"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb"
dependencies = [
"libc",
]
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
@ -610,6 +771,26 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "kqueue"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c"
dependencies = [
"kqueue-sys",
"libc",
]
[[package]]
name = "kqueue-sys"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
dependencies = [
"bitflags 1.3.2",
"libc",
]
[[package]]
name = "libc"
version = "0.2.169"
@ -622,6 +803,17 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
[[package]]
name = "libredox"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.6.0",
"libc",
"redox_syscall",
]
[[package]]
name = "litemap"
version = "0.7.4"
@ -720,6 +912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
dependencies = [
"libc",
"log",
"wasi",
"windows-sys 0.52.0",
]
@ -740,6 +933,34 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "notify"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009"
dependencies = [
"bitflags 2.6.0",
"filetime",
"fsevent-sys",
"inotify",
"kqueue",
"libc",
"log",
"mio",
"notify-types",
"walkdir",
"windows-sys 0.52.0",
]
[[package]]
name = "notify-types"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585d3cb5e12e01aed9e8a1f70d5c6b5e86fe2a6e48fc8cd0b3e0b8df6f6eb174"
dependencies = [
"instant",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@ -866,6 +1087,12 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "ppv-lite86"
version = "0.2.20"
@ -896,7 +1123,7 @@ version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14"
dependencies = [
"bitflags",
"bitflags 2.6.0",
"getopts",
"memchr",
"pulldown-cmark-escape",
@ -954,7 +1181,7 @@ version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
dependencies = [
"bitflags",
"bitflags 2.6.0",
]
[[package]]
@ -998,6 +1225,15 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
@ -1066,6 +1302,15 @@ version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
[[package]]
name = "slab"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
dependencies = [
"autocfg",
]
[[package]]
name = "smallvec"
version = "1.13.2"
@ -1207,6 +1452,17 @@ dependencies = [
"syn",
]
[[package]]
name = "tokio-stream"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "toml"
version = "0.8.19"
@ -1312,21 +1568,35 @@ dependencies = [
"chrono",
"clap",
"compute_graph",
"debounced",
"env_logger",
"futures",
"html5ever",
"log",
"markup5ever_rcdom",
"notify",
"once_cell",
"pulldown-cmark",
"regex",
"serde",
"serde_json",
"tokio",
"tokio-stream",
"toml",
"unicode-normalization",
"url",
]
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
@ -1387,6 +1657,15 @@ version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
[[package]]
name = "winapi-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "windows-core"
version = "0.52.0"

View File

@ -21,15 +21,19 @@ askama = "0.12.1"
chrono = { version = "0.4.39", features = ["serde"] }
clap = { version = "4.5.23", features = ["cargo"] }
compute_graph = { path = "crates/compute_graph" }
debounced = "0.2.0"
env_logger = "0.11.6"
futures = "0.3.31"
html5ever = "0.27.0"
log = "0.4.22"
markup5ever_rcdom = "0.3.0"
notify = "7.0.0"
once_cell = "1.20.2"
pulldown-cmark = "0.12.2"
regex = "1.11.1"
serde = { version = "1.0", features = ["derive"] }
tokio = { version = "1.42.0", features = ["full"] }
tokio-stream = "0.1.17"
toml = "0.8.19"
unicode-normalization = "0.1.24"
url = "2.5.4"

View File

@ -4,27 +4,31 @@ mod posts;
mod tags;
mod util;
use std::cell::RefCell;
use std::rc::Rc;
use compute_graph::{AsyncGraph, builder::GraphBuilder};
use util::{Combine, MapToVoid};
pub async fn generate() -> anyhow::Result<()> {
pub use util::content_base_path;
pub use util::file_watcher::FileWatcher;
pub async fn generate(watcher: Rc<RefCell<FileWatcher>>) -> anyhow::Result<AsyncGraph<()>> {
std::fs::create_dir_all("out").expect("creating output dir");
// TODO: file watching
let mut graph = make_graph()?;
let mut graph = make_graph(watcher)?;
graph.evaluate_async().await;
println!("{}", graph.as_dot_string());
Ok(())
Ok(graph)
}
fn make_graph() -> anyhow::Result<AsyncGraph<()>> {
fn make_graph(watcher: Rc<RefCell<FileWatcher>>) -> anyhow::Result<AsyncGraph<()>> {
let mut builder = GraphBuilder::new_async();
let (void_outputs, posts, all_posts, post_metadatas) = posts::make_graph(&mut builder);
let (void_outputs, posts, all_posts, post_metadatas) = posts::make_graph(&mut builder, watcher);
let archive = archive::make_graph(&mut builder, all_posts);

View File

@ -1,7 +1,9 @@
pub mod content;
pub mod metadata;
use std::cell::RefCell;
use std::path::PathBuf;
use std::rc::Rc;
use askama::Template;
use compute_graph::{
@ -18,19 +20,26 @@ use metadata::PostMetadata;
use crate::generator::util::output_rendered_template;
use super::util::{MapDynamicToVoid, content_path, templates::TemplateCommon};
use super::{
FileWatcher,
util::{MapDynamicToVoid, content_path, templates::TemplateCommon},
};
pub fn make_graph(
builder: &mut GraphBuilder<(), Asynchronous>,
watcher: Rc<RefCell<FileWatcher>>,
) -> (
Input<()>,
DynamicInput<ReadPostOutput>,
Input<Vec<Post<HtmlContent>>>,
Input<Vec<PostMetadata>>,
) {
// todo: make this invalidatable, watch files
let post_files = builder.add_rule(ListPostFiles);
let posts = builder.add_dynamic_rule(MakeReadNodes::new(post_files));
let (post_files, invalidate_posts) = builder.add_invalidatable_rule(ListPostFiles);
watcher.borrow_mut().watch(content_path("posts/"), move || {
invalidate_posts.invalidate();
});
let posts = builder.add_dynamic_rule(MakeReadNodes::new(post_files, watcher));
let extract_metadatas = builder.add_dynamic_rule(MakeExtractMetadatas::new(posts.clone()));
@ -80,12 +89,14 @@ fn find_index(path: PathBuf) -> Option<PathBuf> {
#[derive(InputVisitable)]
struct MakeReadNodes {
files: Input<Vec<PathBuf>>,
watcher: Rc<RefCell<FileWatcher>>,
node_factory: DynamicNodeFactory<PathBuf, ReadPostOutput>,
}
impl MakeReadNodes {
fn new(files: Input<Vec<PathBuf>>) -> Self {
fn new(files: Input<Vec<PathBuf>>, watcher: Rc<RefCell<FileWatcher>>) -> Self {
Self {
files,
watcher,
node_factory: DynamicNodeFactory::new(),
}
}
@ -95,7 +106,11 @@ impl DynamicRule for MakeReadNodes {
fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec<Input<Self::ChildOutput>> {
for file in self.files.value().iter() {
self.node_factory.add_rule(ctx, file.clone(), |ctx| {
ctx.add_rule(ReadPost { path: file.clone() })
let (input, signal) = ctx.add_invalidatable_rule(ReadPost { path: file.clone() });
self.watcher
.borrow_mut()
.watch(file.clone(), move || signal.invalidate());
input
});
}
self.node_factory.all_nodes(ctx)

View File

@ -0,0 +1,80 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
use notify::{EventHandler, Watcher};
use tokio::sync::mpsc::UnboundedSender;
use crate::generator::util::content_base_path;
pub struct FileWatcher {
handlers: HashMap<PathBuf, Box<dyn Fn() -> ()>>,
watcher: Option<notify::RecommendedWatcher>,
}
impl FileWatcher {
pub fn new() -> Self {
Self {
handlers: HashMap::new(),
watcher: None,
}
}
pub fn watch(&mut self, path: PathBuf, f: impl Fn() -> () + 'static) {
if let Some(existing) = self.handlers.remove(&path) {
self.handlers.insert(
path,
Box::new(move || {
existing();
f();
}),
);
} else {
self.handlers.insert(path, Box::new(f));
}
}
pub async fn start(&mut self) -> anyhow::Result<()> {
assert!(self.watcher.is_none());
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<notify::Result<notify::Event>>();
let mut watcher = notify::recommended_watcher(AsyncEventHandler(tx))?;
watcher.watch(&content_base_path(), notify::RecursiveMode::Recursive)?;
self.watcher = Some(watcher);
let mut absolute_content_parent = content_base_path().canonicalize()?;
absolute_content_parent.pop();
while let Some(result) = rx.recv().await {
if let Ok(ev) = result {
self.handle_event(ev, &absolute_content_parent);
}
}
Ok(())
}
fn handle_event(&self, event: notify::Event, base: &Path) {
for path in event.paths {
let relative = path
.strip_prefix(base)
.expect("should only receive events for paths in content path");
let mut path = PathBuf::new();
for component in relative {
path.push(component);
if let Some(handler) = self.handlers.get(&path) {
handler();
}
}
}
}
}
struct AsyncEventHandler(UnboundedSender<notify::Result<notify::Event>>);
impl EventHandler for AsyncEventHandler {
fn handle_event(&mut self, event: notify::Result<notify::Event>) {
self.0.send(event).expect("sending event");
}
}

View File

@ -1,3 +1,4 @@
pub mod file_watcher;
pub mod one_more;
pub mod slugify;
pub mod templates;
@ -80,8 +81,12 @@ pub fn from_frontmatter<D: DeserializeOwned>(contents: &str) -> anyhow::Result<(
Ok((deserialized, chars.as_str()))
}
pub fn content_base_path() -> PathBuf {
PathBuf::from("site_test/")
}
pub fn content_path(p: impl AsRef<Path>) -> PathBuf {
let mut buf = PathBuf::from("site_test/");
let mut buf = content_base_path();
join_abs(&mut buf, p.as_ref());
buf
}

View File

@ -3,7 +3,16 @@
mod generator;
use clap::{Command, arg, command};
use debounced::debounced;
use futures::FutureExt;
use generator::{FileWatcher, content_base_path};
use log::info;
use std::cell::RefCell;
use std::rc::Rc;
use std::time::Duration;
use tokio::pin;
use tokio_stream::StreamExt;
use tokio_stream::wrappers::UnboundedReceiverStream;
#[tokio::main]
async fn main() {
@ -12,7 +21,10 @@ async fn main() {
let matches = command!()
.subcommand_required(true)
.arg_required_else_help(true)
.subcommand(Command::new("gen"))
.subcommand(
Command::new("gen")
.arg(arg!(--watch "Watch the site directory and regenerate on changes")),
)
.subcommand(
Command::new("serve")
.arg(arg!(--watch "Watch the site directory and regenerate on changes")),
@ -26,8 +38,47 @@ async fn main() {
}
match matches.subcommand() {
Some(("gen", _)) => {
generator::generate().await.expect("generating");
Some(("gen", matches)) => {
let watcher = Rc::new(RefCell::new(FileWatcher::new()));
let mut graph = generator::generate(Rc::clone(&watcher))
.await
.expect("generating");
if matches.contains_id("watch") {
let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<()>();
let mut watcher = watcher.borrow_mut();
watcher.watch(content_base_path(), move || {
tx.send(()).expect("sending regenerate signal");
});
let mut debounced =
debounced(UnboundedReceiverStream::new(rx), Duration::from_millis(500));
let watch = watcher.start().fuse();
let regenerate = async move {
while let Some(_) = debounced.next().await {
info!("Regenerating");
graph.evaluate_async().await;
}
}
.fuse();
pin!(regenerate, watch);
loop {
futures::select! {
watcher_res = watch => {
watcher_res.expect("watching files");
}
_ = regenerate => {
info!("regenerate channel closed");
}
}
}
}
}
Some(("serve", _matches)) => {
todo!()