Start on the graph generator implementation

Cargo.toml

@@ -5,9 +5,9 @@ workspace = { members = [
 ] }
 
 [package]
-name = "v6"
+name = "v7"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -16,64 +16,19 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 
 [dependencies]
-activitystreams = "0.7.0-alpha.22"
-activitystreams-ext = "0.1.0-alpha.3"
-ammonia = "3.2"
-anyhow = "1.0"
-askama = "0.11.1"
-axum = "0.5.6"
-base64 = "0.13"
-chrono = { version = "0.4", features = ["serde"] }
-clap = { version = "3.1", features = ["cargo"] }
+anyhow = "1.0.95"
+chrono = { version = "0.4.39", features = ["serde"] }
+clap = { version = "4.5.23", features = ["cargo"] }
 compute_graph = { path = "crates/compute_graph" }
-env_logger = "0.9"
-futures = "0.3"
-html5ever = "0.26"
-http-signature-normalization = "0.6"
-hyper = "0.14"
-log = "0.4"
-markup5ever_rcdom = "0.2"
-mime = "0.3"
-notify = "5.0.0-pre.16"
-notify-debouncer-mini = { version = "*", default-features = false }
-once_cell = "1.13"
-# NOTE: openssl version also needs to be updated in ios target config below
-openssl = "0.10"
-pulldown-cmark = "0.9"
-regex = "1.5"
-reqwest = { version = "0.11", features = ["json"] }
-rsass = "0.25"
-rss = { version = "2.0", features = ["atom"] }
+env_logger = "0.11.6"
+html5ever = "0.27.0"
+log = "0.4.22"
+markup5ever_rcdom = "0.3.0"
+once_cell = "1.20.2"
+pulldown-cmark = "0.12.2"
+regex = "1.11.1"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-splash-rs = { version = "0.3.1", git = "https://git.shadowfacts.net/shadowfacts/splash-rs.git" }
-sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "sqlite"] }
-thiserror = "1.0"
-tokio = { version = "1.18", features = ["full"] }
-tokio-cron-scheduler = "0.8"
-tokio-stream = { version = "0.1.8", features = ["fs"] }
-toml = "0.5"
-tower = "0.4"
-tower-http = { version = "0.3", features = ["fs"] }
-# should be from crates.io
-tree-sitter-bash = { version = "0.20", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-bash.git" }
-tree-sitter-c = "0.20"
-# should be https://github.com/tree-sitter/tree-sitter-css.git
-tree-sitter-css = { version = "0.20", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-css.git" }
-tree-sitter-elixir = { version = "0.19", git = "https://github.com/elixir-lang/tree-sitter-elixir.git" }
-tree-sitter-highlight = "0.20"
-# should be from crates.io
-tree-sitter-html = { version = "0.20", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-html.git" }
-# should be from crates.io
-tree-sitter-java = { version = "0.20", git = "https://github.com/tree-sitter/tree-sitter-java.git" }
-tree-sitter-javascript = "0.20"
-tree-sitter-json = { version = "0.20", git = "https://github.com/tree-sitter/tree-sitter-json.git" }
-# should be https://github.com/jiyee/tree-sitter-objc.git
-tree-sitter-objc = { version = "1.0.0", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-objc.git" }
-tree-sitter-rust = "0.20"
-unicode-normalization = "0.1.19"
-url = "2.2"
-uuid = { version = "1.1", features = ["v4"] }
-
-[target.'cfg(target_os = "ios")'.dependencies]
-openssl = { version = "0.10", features = ["vendored"] }
+tokio = { version = "1.42.0", features = ["full"] }
+toml = "0.8.19"
+unicode-normalization = "0.1.24"
+url = "2.5.4"

Cargo.toml.bak

@@ -0,0 +1,79 @@
+workspace = { members = [
+    "crates/compute_graph",
+    "crates/compute_graph_macros",
+    "crates/derive_test",
+] }
+
+[package]
+name = "v6"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[build-dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+[dependencies]
+activitystreams = "0.7.0-alpha.22"
+activitystreams-ext = "0.1.0-alpha.3"
+ammonia = "3.2"
+anyhow = "1.0"
+askama = "0.11.1"
+axum = "0.5.6"
+base64 = "0.13"
+chrono = { version = "0.4", features = ["serde"] }
+clap = { version = "3.1", features = ["cargo"] }
+compute_graph = { path = "crates/compute_graph" }
+env_logger = "0.9"
+futures = "0.3"
+html5ever = "0.26"
+http-signature-normalization = "0.6"
+hyper = "0.14"
+log = "0.4"
+markup5ever_rcdom = "0.2"
+mime = "0.3"
+notify = "5.0.0-pre.16"
+notify-debouncer-mini = { version = "*", default-features = false }
+once_cell = "1.13"
+# NOTE: openssl version also needs to be updated in ios target config below
+openssl = "0.10"
+pulldown-cmark = "0.9"
+regex = "1.5"
+reqwest = { version = "0.11", features = ["json"] }
+rsass = "0.25"
+rss = { version = "2.0", features = ["atom"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+splash-rs = { version = "0.3.1", git = "https://git.shadowfacts.net/shadowfacts/splash-rs.git" }
+sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "sqlite"] }
+thiserror = "1.0"
+tokio = { version = "1.18", features = ["full"] }
+tokio-cron-scheduler = "0.8"
+tokio-stream = { version = "0.1.8", features = ["fs"] }
+toml = "0.5"
+tower = "0.4"
+tower-http = { version = "0.3", features = ["fs"] }
+# should be from crates.io
+tree-sitter-bash = { version = "0.20", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-bash.git" }
+tree-sitter-c = "0.20"
+# should be https://github.com/tree-sitter/tree-sitter-css.git
+tree-sitter-css = { version = "0.20", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-css.git" }
+tree-sitter-elixir = { version = "0.19", git = "https://github.com/elixir-lang/tree-sitter-elixir.git" }
+tree-sitter-highlight = "0.20"
+# should be from crates.io
+tree-sitter-html = { version = "0.20", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-html.git" }
+# should be from crates.io
+tree-sitter-java = { version = "0.20", git = "https://github.com/tree-sitter/tree-sitter-java.git" }
+tree-sitter-javascript = "0.20"
+tree-sitter-json = { version = "0.20", git = "https://github.com/tree-sitter/tree-sitter-json.git" }
+# should be https://github.com/jiyee/tree-sitter-objc.git
+tree-sitter-objc = { version = "1.0.0", git = "https://git.shadowfacts.net/shadowfacts/tree-sitter-objc.git" }
+tree-sitter-rust = "0.20"
+unicode-normalization = "0.1.19"
+url = "2.2"
+uuid = { version = "1.1", features = ["v4"] }
+
+[target.'cfg(target_os = "ios")'.dependencies]
+openssl = { version = "0.10", features = ["vendored"] }

build.rs

@@ -2,9 +2,6 @@ use serde::Deserialize;
 use std::process::Command;
 
 fn main() {
-    // trigger recompilation when a new migration is added
-    println!("cargo:rerun-if-changed=migrations");
-
     let target = get_swift_target_info();
     if target.target.unversioned_triple.contains("linux") {
         target.paths.runtime_library_paths.iter().for_each(|path| {

crates/compute_graph/src/rule.rs

@@ -182,6 +182,11 @@ impl<T> Input<T> {
                 .expect("node must be evaluated before reading value")
         })
     }
+
+    /// Retrieves the ID of the node that this input refers to.
+    pub fn node_id(&self) -> NodeId {
+        self.node_idx.get().unwrap()
+    }
 }
 
 // can't derive this impl because it incorectly adds the bound T: Clone

site_test/posts/2024-11-30-swiftui-lifecycle.md

@@ -0,0 +1,74 @@
+```
+title = "Your View's Lifetime Is Not Yours"
+tags = ["swift"]
+date = "2024-11-30 18:41:42 -0500"
+slug = "swiftui-lifecycle"
+```
+
+When SwiftUI was announced in 2019 (oh boy, more than 5 years ago), one of the big things that the Apple engineers emphasized was that a SwiftUI `View` is not like a UIKit/AppKit view. As Apple was at pains to note, SwiftUI may evaluate the `body` property of a `View` arbitrarily often. It's easy to miss an important consequence this has, though: your `View` will also be initilaized arbitrarily often. This is why SwiftUI views are supposed to be structs which are simple value types and can be stack-allocated: constructing a `View` needs to be cheap.
+
+More precisely, what I mean by the title of this post is that the lifetime of a struct that conforms to `View` is unmoored from that of the conceptual thing representing a piece of your user interface. 
+
+<!-- excerpt-end -->
+
+<aside class="inline">
+
+A note on definitions: in this post, I'm using the word 'lifecycle' and 'lifetime' in very particular ways. Lifecycle is connected to the conceptual view, and encompasses things like the `onAppear`/`onDisappear` modifiers. Lifetime, however, is a property of a value: the time from its initialization to its destruction (that is, how long the value exists in memory).
+
+</aside>
+
+
+
+This comes up on social media with some regularity. Every few months there'll be questions about (either directly, or in the form of questions that boil down to) why some view's initializer is being called more than expected. One particularly common case is people migrating from Combine/`ObservableObject` to the iOS 17+ `@Observable`. 
+
+If you were not paying attention to SwiftUI in the early days, or have only started learning it more recently, there are some important details about the nature of the framework you might have missed. One of the things the SwiftUI engineers emphasized when it was announced was that the `body` property can be called at any time by the framework and may be called as often as the framework likes. What follows from this—that's not entirely obvious or always explicitly stated—is that `View` initializers run with the same frequency. Consider the minimal possible case:
+
+```swift
+struct MyParentView: View {
+    var body: some View {
+        MyChildView()
+    }
+}
+```
+
+Evaluating `MyParentView`'s body necessarily means running the initializer for `MyChildView`. SwiftUI does a lot of work to make things feel magical, but, at the end of the day, the code that you're writing is ultimately actually running. So, the initializer runs.
+
+After that happens, SwiftUI can compare the new and old values of `MyChildView` to decide whether it's changed and, in turn, whether to read its body. But, by the time the framework is making that decision, the initializer has already been run.
+
+## An Autoclosure Corollary
+
+It follows, I argue, from the reasons given above that it is a poor API design choice for SwiftUI property wrappers to use `@autoclosure` parameters in the wrapped value initializer.
+
+An entirely reasonable principle of API design is that you should hide unnecessary complexity from consumers of your API. There is, however, a very fine line between hiding unnecessary complexity and obscuring necessary complexity—or worse, being actively misleading about what's happening. I think `@StateObject` and its autoclosure initializer tread too close to that line.
+
+The use of an autoclosure hides the fact that the property wrapper itself is getting initialized frequently—just like the view that contains it (initializing the view necessarily means initializing all of its properties—i.e., the property wrappers themselves). If your API design is such that you can very easily write two pieces code that, on the surface, look almost identical but have dramatically different performance characteristics, then it hinders (especially inexperienced) developers' understanding.
+
+```swift
+@StateObject var model = MyViewModel()
+// vs.
+@State var model = MyViewModel()
+```
+
+That the name `StateObject` conflates the behavior with `State` certainly doesn't help either.
+
+## Some Recommendations
+
+Not realizing that view initializers—not just the `body` property—are in the hot path can quickly get you into hot water, particularly if you're doing any expensive work or are trying to use RAII[^1] with a value in a view property. So, what follows are some general recommendations for avoiding those issues that are as close to universally correct as I'm willing to claim in writing.
+
+[^1]: Resource Acquisition Is Initialization: the idea that the lifetime of a value is tied to some other resource (e.g., an open file). The wrapped value of a `@State` property is evaluated every time the containing view is initialized, so doing any resource acquisition work therein can be expensive.
+
+### No Expensive Work in Inits
+
+This is a simple one. As discussed earlier, this is equivalent to doing that same expensive work in a view's `body` property. But SwiftUI calls both of those at arbitrary times with arbitrary frequency. Doing expensive work in either place will quickly cause performance issues.
+
+### Ignore `View` Lifetimes
+
+Even more generally, you should ignore the lifetime of your `View` structs altogether. "When is my view initialized? Who knows, I don't care!" Even if, at some point during development, you find that a particular view's initializer is called at a useful time, avoid depending on that, even if the work you need to do is cheap. While, right now, you might have a good handle on a particular `View`'s lifetime, it's very easy to inadvertently change that in the future, either by introducing additional dependencies in the view's parent, due to changes to the [view's identity](https://developer.apple.com/videos/play/wwdc2021/10022/), or due to entirely framework-internal changes.
+
+Note that this is a particularly important idea and is worth hammering home: a `View`'s initializer is called as part of its _parent's_ `body`. This is particularly prone to action at a distance: seemingly-unrelated changes at the callsite where a view is used (or beyond) can dramatically change how often that view is initialized.
+
+### Use Lifecycle Modifiers
+
+The [`@State` docs](https://developer.apple.com/documentation/swiftui/state#Store-observable-objects) call this out with regard to `Observable`, but it's true in general. Rather than putting expensive objects as the wraped value of a `@State` property wrapper, use SwiftUI's lifecycle modifiers like `.onAppear` or `.task` to construct them.
+
+What's more, if you need to do any one-time setup work—even inexpensive—related to a particular instance of a _conceptual_ view, the lifecycle modifiers are the way to go. They abstract you away from the particularities of the lifetime of the struct instance itself.

src.bak/generator/mod.rs

@@ -0,0 +1,76 @@
+pub mod archive;
+mod copy;
+mod css;
+mod highlight;
+mod home;
+mod markdown;
+mod pagination;
+pub mod posts;
+mod rss;
+mod tags;
+mod tutorials;
+mod tv;
+pub mod util;
+
+pub use crate::generator::posts::parse::{HtmlContent, Post};
+use std::path::{Component, Path, PathBuf};
+
+pub async fn generate() -> anyhow::Result<Vec<Post<HtmlContent>>> {
+    std::fs::create_dir_all("out").expect("creating output dir");
+
+    let posts = posts::parse().await?;
+    let post_refs = posts.iter().collect::<Vec<_>>();
+    let tags = tags::generate(&posts);
+    posts::generate(&posts)?;
+    home::generate(&post_refs)?;
+    rss::generate(&posts, &tags);
+    archive::generate(&posts);
+
+    css::generate()?;
+
+    copy::copy();
+
+    tv::generate();
+    tutorials::generate();
+
+    Ok(posts)
+}
+
+pub fn content_path(p: impl AsRef<Path>) -> PathBuf {
+    let mut buf = PathBuf::from("site/");
+    join_abs(&mut buf, p.as_ref());
+    buf
+}
+
+pub fn output_path(p: impl AsRef<Path>) -> PathBuf {
+    let mut buf = PathBuf::from("out/");
+    join_abs(&mut buf, p.as_ref());
+    buf
+}
+
+fn join_abs(buf: &mut PathBuf, p: &Path) {
+    for comp in p.components() {
+        match comp {
+            Component::RootDir => (),
+            Component::CurDir => (),
+            Component::Normal(c) => buf.push(c),
+            Component::Prefix(_) => panic!("prefixes are unsupported"),
+            Component::ParentDir => {
+                buf.pop();
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::join_abs;
+    use std::path::{Path, PathBuf};
+
+    #[test]
+    fn test_join_abs() {
+        let mut buf = PathBuf::from("site/");
+        join_abs(&mut buf, Path::new("/2022/test/"));
+        assert_eq!(buf, PathBuf::from("site/2022/test/"));
+    }
+}

src.bak/generator/posts.rs

@@ -0,0 +1,5 @@
+pub mod generate;
+pub mod parse;
+
+pub use generate::generate;
+pub use parse::{parse, HtmlContent, Post};

src.bak/generator/util/mod.rs

@@ -0,0 +1,80 @@
+pub mod one_more;
+pub mod slugify;
+pub mod templates;
+pub mod word_count;
+
+use anyhow::anyhow;
+use askama::Template;
+use serde::Deserialize;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use std::path::Path;
+
+pub fn output_writer(path: impl AsRef<Path>) -> Result<impl Write, std::io::Error> {
+    let path = crate::generator::output_path(path);
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+    let file = File::create(path)?;
+    Ok(BufWriter::new(file))
+}
+
+pub fn output_rendered_template(
+    template: &impl Template,
+    file: impl AsRef<Path>,
+) -> Result<(), std::io::Error> {
+    let path = file.as_ref();
+    let writer = output_writer(path)?;
+    template.render_into(&mut FmtWriter(writer)).map_err(|e| {
+        std::io::Error::new(
+            std::io::ErrorKind::Other,
+            format!("writing template {}: {}", path.display(), e),
+        )
+    })
+}
+
+struct FmtWriter<W: Write>(W);
+
+impl<W: Write> std::fmt::Write for FmtWriter<W> {
+    fn write_str(&mut self, s: &str) -> std::fmt::Result {
+        self.0.write_all(s.as_bytes()).map_err(|_| std::fmt::Error)
+    }
+
+    fn write_char(&mut self, c: char) -> std::fmt::Result {
+        let mut buf = [0u8; 4];
+        c.encode_utf8(&mut buf);
+        self.0.write_all(&buf).map_err(|_| std::fmt::Error)
+    }
+
+    fn write_fmt(&mut self, args: std::fmt::Arguments<'_>) -> std::fmt::Result {
+        self.0.write_fmt(args).map_err(|_| std::fmt::Error)
+    }
+}
+
+pub fn from_frontmatter<'de, D: Deserialize<'de>>(
+    contents: &'de str,
+) -> anyhow::Result<(D, &'de str)> {
+    let mut chars = contents.char_indices();
+    for i in 0..=2 {
+        if chars.next() != Some((i, '`')) {
+            return Err(anyhow!("no frontmatter"));
+        }
+    }
+    let mut seen_backticks = 0;
+    let mut end_index = None;
+    while let Some((idx, c)) = chars.next() {
+        if c == '`' {
+            seen_backticks += 1;
+            if seen_backticks == 3 {
+                end_index = Some(idx - 3);
+                break;
+            }
+        } else {
+            seen_backticks = 0;
+        }
+    }
+    let end_index = end_index.ok_or(anyhow!("missing frontmatter end"))?;
+    let frontmatter = &contents[3..=end_index];
+    let deserialized = toml::from_str::<'de, D>(frontmatter)?;
+    Ok((deserialized, chars.as_str()))
+}

src.bak/main.rs

@@ -0,0 +1,232 @@
+#![feature(let_chains)]
+
+mod activitypub;
+mod generator;
+mod graph_generator;
+
+use crate::generator::{HtmlContent, Post};
+use axum::{
+    body::{boxed, Body, Bytes},
+    http::{Request, StatusCode},
+    response::{Html, IntoResponse, Redirect, Response},
+    routing::{get, get_service},
+    Extension, Router,
+};
+use clap::{arg, command, Command};
+use generator::output_path;
+use log::error;
+use log::info;
+use notify_debouncer_mini::new_debouncer;
+use once_cell::sync::Lazy;
+use sqlx::{
+    sqlite::{SqliteConnectOptions, SqlitePoolOptions},
+    ConnectOptions, SqlitePool,
+};
+use std::path::Path;
+use std::{convert::Infallible, net::SocketAddr};
+use tokio::io::AsyncReadExt;
+use tokio_cron_scheduler::{Job, JobScheduler};
+use tower::Service;
+use tower_http::services::{ServeDir, ServeFile};
+
+#[tokio::main]
+async fn main() {
+    env_logger::init();
+
+    let matches = command!()
+        .subcommand_required(true)
+        .arg_required_else_help(true)
+        .subcommand(Command::new("graph-gen"))
+        .subcommand(Command::new("gen"))
+        .subcommand(
+            Command::new("serve")
+                .arg(arg!(--watch "Watch the site directory and regenerate on changes")),
+        )
+        .get_matches();
+
+    if cfg!(debug_assertions) {
+        info!("Running in dev mode");
+    } else {
+        info!("Running in release mode");
+    }
+
+    match matches.subcommand() {
+        Some(("graph-gen", _)) => {
+            graph_generator::generate().await.unwrap();
+        }
+        Some(("gen", _)) => {
+            let _ = generate().await;
+        }
+        Some(("serve", matches)) => {
+            // ensure that the keys are loadable
+            _ = Lazy::force(&activitypub::keys::PUB_KEY_PEM);
+            _ = Lazy::force(&activitypub::keys::PRIV_KEY_PEM);
+
+            let posts = generate().await.expect("initial generation");
+            info!("Generated");
+
+            let pool = setup_db().await;
+            activitypub::articles::insert_posts(&posts, &pool).await;
+
+            if matches.is_present("watch") {
+                start_watcher();
+            }
+
+            let pool_ = pool.clone();
+            tokio::spawn(async move {
+                match activitypub::articles::federate_outgoing(&pool_).await {
+                    Ok(()) => (),
+                    Err(e) => error!("Federating outgoing articles: {:?}", e),
+                }
+            });
+
+            let sched = JobScheduler::new().await.expect("create JobScheduler");
+            let digest_schedule = if cfg!(debug_assertions) {
+                // every 5 minutes in debug
+                "0 1/5 * * * *"
+            } else {
+                // every day at midnight utc
+                "0 0 0 * * * *"
+            };
+            let pool_ = pool.clone();
+            sched
+                .add(
+                    Job::new_async(digest_schedule, move |_, _| {
+                        // this closure executes multiple times, so we need to clone the pool every
+                        // time rather than moving it into the closure
+                        let pool = pool_.clone();
+                        Box::pin(async move {
+                            activitypub::digester::send_digest_if_necessary(&pool).await;
+                        })
+                    })
+                    .expect("creating digest job"),
+                )
+                .await
+                .expect("adding digest job");
+            sched.start().await.expect("starting JobScheduler");
+
+            serve(&posts, pool).await;
+        }
+        _ => unreachable!(),
+    }
+}
+
+async fn generate() -> anyhow::Result<Vec<Post<HtmlContent>>> {
+    generator::generate().await
+}
+
+fn start_watcher() {
+    let handle = tokio::runtime::Handle::current();
+
+    std::thread::spawn(move || {
+        let (tx, rx) = std::sync::mpsc::channel();
+
+        let mut debouncer =
+            new_debouncer(std::time::Duration::from_millis(100), None, tx).expect("debouncer");
+
+        debouncer
+            .watcher()
+            .watch(Path::new("site/"), notify::RecursiveMode::Recursive)
+            .expect("watch");
+
+        info!("Started watcher");
+
+        for events in rx {
+            let events = events.unwrap();
+            let paths = events.iter().map(|ev| &ev.path).collect::<Vec<_>>();
+            info!("Regenerating due to changes at {:?}", paths);
+            handle.spawn(async move {
+                match generate().await {
+                    Ok(_) => (),
+                    Err(e) => error!("Regeneration failed: {:?}", e),
+                }
+            });
+        }
+    });
+}
+
+async fn setup_db() -> SqlitePool {
+    let mut options = SqliteConnectOptions::new()
+        .filename(std::env::var("DB_PATH").unwrap_or("db.sqlite".to_owned()));
+    options.log_statements(log::LevelFilter::Debug);
+    let pool = SqlitePoolOptions::new()
+        .connect_with(options)
+        .await
+        .unwrap();
+
+    sqlx::migrate!()
+        .run(&pool)
+        .await
+        .expect("database migrated");
+
+    pool
+}
+
+async fn serve(posts: &[Post<HtmlContent>], pool: SqlitePool) {
+    let articles_or_fallback = tower::service_fn(handle_ap_article_or_serve_dir);
+
+    let app = Router::new()
+        .merge(activitypub::router(pool.clone()))
+        .merge(redirect_router(posts))
+        .fallback(get_service(articles_or_fallback))
+        .layer(Extension(pool));
+
+    let addr = if cfg!(debug_assertions) {
+        SocketAddr::from(([0, 0, 0, 0], 8084))
+    } else {
+        SocketAddr::from(([127, 0, 0, 1], 8084))
+    };
+    info!("Listening on {}", addr);
+    axum::Server::bind(&addr)
+        .serve(app.into_make_service())
+        .await
+        .unwrap();
+}
+
+async fn handle_ap_article_or_serve_dir(req: Request<Body>) -> Result<Response, Infallible> {
+    let articles_res = activitypub::articles::handle(&req).await;
+
+    match articles_res {
+        Some(res) => Ok(res.into_response()),
+        None => {
+            let mut serve_dir =
+                ServeDir::new("out").fallback(ServeFile::new(output_path("404.html")));
+            match serve_dir.call(req).await {
+                Ok(resp) => Ok(resp.map(boxed)),
+                Err(e) => Ok(handle_error(e).await.into_response()),
+            }
+        }
+    }
+}
+
+fn redirect_router(posts: &[Post<HtmlContent>]) -> Router {
+    let mut r = Router::new();
+    for post in posts.iter() {
+        for path in post.metadata.old_permalink.iter().flatten() {
+            let new_permalink = post.permalink();
+            r = r.route(
+                path,
+                get(|_: Request<Body>| async move { Redirect::permanent(&new_permalink) }),
+            );
+        }
+    }
+    r
+}
+
+async fn handle_error(err: std::io::Error) -> impl IntoResponse {
+    error!("Unhandled error: {}", err);
+    (
+        StatusCode::INTERNAL_SERVER_ERROR,
+        serve_500().await.unwrap_or_else(|err| {
+            error!("Unhandled error serving 500 page: {}", &err);
+            "Internal server error".into_response()
+        }),
+    )
+}
+
+async fn serve_500() -> Result<Response, std::io::Error> {
+    let mut buf = vec![];
+    let mut f = tokio::fs::File::open(output_path("500.html")).await?;
+    f.read_to_end(&mut buf).await?;
+    Ok(Html(Bytes::from(buf)).into_response())
+}

src/generator/markdown.rs

@@ -4,11 +4,11 @@ mod heading_anchors;
 mod highlight;
 mod link_decorations;
 
-use pulldown_cmark::{html, Event, Options, Parser};
+use pulldown_cmark::{Event, Options, Parser, html};
 use std::io::Write;
 
 pub fn render(s: &str, writer: impl Write) {
-    html::write_html(writer, parse(s)).unwrap();
+    html::write_html_io(writer, parse(s)).unwrap();
 }
 
 pub fn parse<'a>(s: &'a str) -> impl Iterator<Item = Event<'a>> {

src/generator/markdown/footnote_backrefs.rs

@@ -1,15 +1,17 @@
-use pulldown_cmark::{CowStr, Event, Tag};
+use pulldown_cmark::{CowStr, Event, Tag, TagEnd};
 use std::iter::Peekable;
 
 pub struct FootnoteBackrefs<'a, I: Iterator<Item = Event<'a>>> {
     iter: Peekable<I>,
     next: Option<Event<'a>>,
+    last_footnote_definition_start: Option<CowStr<'a>>,
 }
 
 pub fn new<'a, I: Iterator<Item = Event<'a>>>(iter: I) -> FootnoteBackrefs<'a, I> {
     FootnoteBackrefs {
         iter: iter.peekable(),
         next: None,
+        last_footnote_definition_start: None,
     }
 }
 
@@ -29,17 +31,25 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for FootnoteBackrefs<'a, I> {
                 );
                 Some(Event::Html(CowStr::Boxed(html.into_boxed_str())))
             }
-            Some(Event::End(Tag::Paragraph)) => {
-                if let Some(Event::End(Tag::FootnoteDefinition(label))) = self.iter.peek() {
+            Some(Event::Start(Tag::FootnoteDefinition(label))) => {
+                self.last_footnote_definition_start = Some(label.clone());
+                Some(Event::Start(Tag::FootnoteDefinition(label)))
+            }
+            Some(Event::End(TagEnd::Paragraph)) => {
+                if let Some(Event::End(TagEnd::FootnoteDefinition)) = self.iter.peek() {
                     assert!(self.next.is_none());
-                    self.next = Some(Event::End(Tag::Paragraph));
+                    self.next = Some(Event::End(TagEnd::Paragraph));
+                    let label = self
+                        .last_footnote_definition_start
+                        .take()
+                        .expect("footnote definition must have started before ending");
                     let html = format!(
                         r##" <a href="#fnref{}" class="footnote-backref">↩</a>"##,
                         label
                     );
                     Some(Event::Html(CowStr::Boxed(html.into_boxed_str())))
                 } else {
-                    Some(Event::End(Tag::Paragraph))
+                    Some(Event::End(TagEnd::Paragraph))
                 }
             }
             e => e,
@@ -49,7 +59,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for FootnoteBackrefs<'a, I> {
 
 #[cfg(test)]
 mod tests {
-    use pulldown_cmark::{html, Options, Parser};
+    use pulldown_cmark::{Options, Parser, html};
 
     fn render(s: &str) -> String {
         let mut out = String::new();

src/generator/markdown/footnote_defs.rs

@@ -1,4 +1,4 @@
-use pulldown_cmark::{CowStr, Event, Tag};
+use pulldown_cmark::{CowStr, Event, Tag, TagEnd};
 use std::collections::VecDeque;
 
 pub fn new<'a, I: Iterator<Item = Event<'a>>>(iter: I) -> FootnoteDefs<'a, I> {
@@ -31,7 +31,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for FootnoteDefs<'a, I> {
 
                 loop {
                     match self.iter.next() {
-                        Some(Event::End(Tag::FootnoteDefinition(_))) => {
+                        Some(Event::End(TagEnd::FootnoteDefinition)) => {
                             self.footnote_events.push_back(Event::Html("</div>".into()));
                             break;
                         }
@@ -63,7 +63,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for FootnoteDefs<'a, I> {
 
 #[cfg(test)]
 mod tests {
-    use pulldown_cmark::{html, Options, Parser};
+    use pulldown_cmark::{Options, Parser, html};
 
     fn render(s: &str) -> String {
         let mut out = String::new();

src/generator/markdown/heading_anchors.rs

@@ -2,8 +2,8 @@ use std::iter::empty;
 
 use crate::generator::util::one_more::OneMore;
 use crate::generator::util::slugify::slugify_iter;
-use pulldown_cmark::{html, CowStr, Event, HeadingLevel, Tag};
 use State::*;
+use pulldown_cmark::{CowStr, Event, HeadingLevel, Tag, TagEnd, html};
 
 pub struct HeadingAnchors<'a, I: Iterator<Item = Event<'a>>> {
     iter: I,
@@ -37,7 +37,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for HeadingAnchors<'a, I> {
 impl<'a, I: Iterator<Item = Event<'a>>> HeadingAnchors<'a, I> {
     fn handle_upstream_event(&mut self, e: Event<'a>) -> Event<'a> {
         match e {
-            Event::Start(Tag::Heading(level, _, _)) => {
+            Event::Start(Tag::Heading { level, .. }) => {
                 let inside_heading = self.accumulate_heading_contents();
                 let slug = slugify_events(&inside_heading);
                 let mut inner = String::new();
@@ -60,7 +60,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> HeadingAnchors<'a, I> {
     fn accumulate_heading_contents(&mut self) -> Vec<Event<'a>> {
         let mut events = vec![];
         while let Some(e) = self.iter.next() {
-            if let Event::End(Tag::Heading(_, _, _)) = e {
+            if let Event::End(TagEnd::Heading(_)) = e {
                 break;
             } else {
                 events.push(e);
@@ -100,7 +100,7 @@ enum State {
 
 #[cfg(test)]
 mod tests {
-    use pulldown_cmark::{html, CowStr, Event, Parser};
+    use pulldown_cmark::{CowStr, Event, Parser, html};
 
     fn render(s: &str) -> String {
         let mut out = String::new();

src/generator/markdown/highlight.rs

@@ -1,5 +1,5 @@
-use crate::generator::highlight::highlight;
-use pulldown_cmark::{CodeBlockKind, Event, Tag};
+// use crate::generator::highlight::highlight;
+use pulldown_cmark::{CodeBlockKind, Event, Tag, TagEnd};
 
 pub fn new<'a, I: Iterator<Item = Event<'a>>>(iter: I) -> Highlight<'a, I> {
     Highlight { iter }
@@ -16,7 +16,8 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Highlight<'a, I> {
         match self.iter.next() {
             Some(Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(label)))) => {
                 let code = self.consume_code_block();
-                let mut highlighted = highlight(&code, &label);
+                // let mut highlighted = highlight(&code, &label);
+                let mut highlighted = "TODO".to_owned();
                 highlighted.insert_str(
                     0,
                     &format!("<pre class=\"highlight\" data-lang=\"{}\"><code>", label),
@@ -34,7 +35,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Highlight<'a, I> {
         let mut buf = String::new();
         loop {
             match self.iter.next() {
-                Some(Event::End(Tag::CodeBlock(CodeBlockKind::Fenced(_)))) => {
+                Some(Event::End(TagEnd::CodeBlock)) => {
                     break;
                 }
                 Some(Event::Text(s)) => {
@@ -49,7 +50,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Highlight<'a, I> {
 
 #[cfg(test)]
 mod tests {
-    use pulldown_cmark::{html, Parser};
+    use pulldown_cmark::{Parser, html};
 
     fn render(s: &str) -> String {
         let mut out = String::new();

src/generator/markdown/link_decorations.rs

@@ -1,4 +1,4 @@
-use pulldown_cmark::{html, CowStr, Event, Tag};
+use pulldown_cmark::{CowStr, Event, Tag, TagEnd, html};
 use url::Url;
 
 pub struct LinkDecorations<'a, I: Iterator<Item = Event<'a>>> {
@@ -14,9 +14,11 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for LinkDecorations<'a, I> {
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.iter.next() {
-            Some(Event::Start(Tag::Link(_, href, title))) => {
-                let link = href_to_pretty_link(&href);
-                let mut s = format!(r#"<a href="{}" data-link="{}"#, href, link);
+            Some(Event::Start(Tag::Link {
+                dest_url, title, ..
+            })) => {
+                let link = href_to_pretty_link(&dest_url);
+                let mut s = format!(r#"<a href="{}" data-link="{}"#, dest_url, link);
                 if !title.is_empty() {
                     s.push_str(" title=\"");
                     s.push_str(title.as_ref());
@@ -39,7 +41,7 @@ impl<'a, 'b, I: Iterator<Item = Event<'a>>> Iterator for LinkContents<'a, 'b, I>
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.0.next() {
-            Some(Event::End(Tag::Link(_, _, _))) => None,
+            Some(Event::End(TagEnd::Link)) => None,
             e => e,
         }
     }
@@ -83,7 +85,7 @@ fn href_to_pretty_link<'a>(href: &CowStr<'a>) -> CowStr<'a> {
 
 #[cfg(test)]
 mod tests {
-    use pulldown_cmark::{html, CowStr, Parser};
+    use pulldown_cmark::{CowStr, Parser, html};
 
     fn pretty_link<'a>(s: &'a str) -> CowStr<'a> {
         super::href_to_pretty_link(&CowStr::Borrowed(s))

src/generator/mod.rs

@@ -1,76 +1,45 @@
-pub mod archive;
-mod copy;
-mod css;
-mod highlight;
-mod home;
 mod markdown;
-mod pagination;
-pub mod posts;
-mod rss;
-mod tags;
-mod tutorials;
-mod tv;
-pub mod util;
+mod posts;
+mod util;
 
-pub use crate::generator::posts::parse::{HtmlContent, Post};
-use std::path::{Component, Path, PathBuf};
+use compute_graph::{
+    AsyncGraph,
+    builder::GraphBuilder,
+    rule::{Input, InputVisitable, Rule},
+};
+use util::MapToVoid;
 
-pub async fn generate() -> anyhow::Result<Vec<Post<HtmlContent>>> {
+pub async fn generate() -> anyhow::Result<()> {
     std::fs::create_dir_all("out").expect("creating output dir");
 
-    let posts = posts::parse().await?;
-    let post_refs = posts.iter().collect::<Vec<_>>();
-    let tags = tags::generate(&posts);
-    posts::generate(&posts)?;
-    home::generate(&post_refs)?;
-    rss::generate(&posts, &tags);
-    archive::generate(&posts);
+    // TODO: file watching
 
-    css::generate()?;
+    let mut graph = make_graph()?;
 
-    copy::copy();
+    graph.evaluate_async().await;
 
-    tv::generate();
-    tutorials::generate();
+    println!("{}", graph.as_dot_string());
 
-    Ok(posts)
+    Ok(())
 }
 
-pub fn content_path(p: impl AsRef<Path>) -> PathBuf {
-    let mut buf = PathBuf::from("site/");
-    join_abs(&mut buf, p.as_ref());
-    buf
+fn make_graph() -> anyhow::Result<AsyncGraph<()>> {
+    let mut builder = GraphBuilder::new_async();
+
+    let (posts, post_metadatas) = posts::make_graph(&mut builder);
+
+    let posts_output = builder.add_rule(MapToVoid(post_metadatas));
+    builder.set_output(Output {
+        posts: posts_output,
+    });
+    Ok(builder.build()?)
 }
 
-pub fn output_path(p: impl AsRef<Path>) -> PathBuf {
-    let mut buf = PathBuf::from("out/");
-    join_abs(&mut buf, p.as_ref());
-    buf
+#[derive(InputVisitable)]
+struct Output {
+    posts: Input<()>,
 }
-
-fn join_abs(buf: &mut PathBuf, p: &Path) {
-    for comp in p.components() {
-        match comp {
-            Component::RootDir => (),
-            Component::CurDir => (),
-            Component::Normal(c) => buf.push(c),
-            Component::Prefix(_) => panic!("prefixes are unsupported"),
-            Component::ParentDir => {
-                buf.pop();
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::join_abs;
-    use std::path::{Path, PathBuf};
-
-    #[test]
-    fn test_join_abs() {
-        let mut buf = PathBuf::from("site/");
-        join_abs(&mut buf, Path::new("/2022/test/"));
-        assert_eq!(buf, PathBuf::from("site/2022/test/"));
-    }
+impl Rule for Output {
+    type Output = ();
+    fn evaluate(&mut self) -> Self::Output {}
 }

src/generator/posts.rs

@@ -1,5 +1,223 @@
-pub mod generate;
-pub mod parse;
+mod content;
+mod metadata;
 
-pub use generate::generate;
-pub use parse::{parse, HtmlContent, Post};
+use std::{collections::HashMap, path::PathBuf};
+
+use compute_graph::{
+    builder::GraphBuilder,
+    rule::{DynamicInput, DynamicRule, DynamicRuleContext, Input, InputVisitable, Rule},
+    synchronicity::Asynchronous,
+};
+use content::{HtmlContent, Post};
+use log::error;
+use metadata::PostMetadata;
+
+use super::util::content_path;
+
+pub fn make_graph(
+    builder: &mut GraphBuilder<(), Asynchronous>,
+) -> (Input<Vec<Post<HtmlContent>>>, Input<Vec<PostMetadata>>) {
+    // todo: make this invalidatable, watch files
+    let post_files = builder.add_rule(ListPostFiles);
+    let posts = builder.add_dynamic_rule(MakeReadNodes::new(post_files));
+
+    let extract_metadatas = builder.add_dynamic_rule(MakeExtractMetadatas::new(posts.clone()));
+
+    (
+        builder.add_rule(AllPosts(posts)),
+        builder.add_rule(AllMetadatas(extract_metadatas)),
+    )
+}
+
+#[derive(InputVisitable)]
+struct ListPostFiles;
+impl Rule for ListPostFiles {
+    type Output = Vec<PathBuf>;
+    fn evaluate(&mut self) -> Self::Output {
+        let posts_path = content_path("posts/");
+        let entries = std::fs::read_dir(posts_path);
+        match entries {
+            Ok(entries) => entries
+                .flat_map(|ent| ent.ok())
+                .map(|ent| {
+                    if ent.file_type().unwrap().is_dir() {
+                        find_index(ent.path()).expect("folder posts must have index file")
+                    } else {
+                        ent.path()
+                    }
+                })
+                .collect(),
+            Err(e) => {
+                error!("Error listing posts: {e:?}");
+                vec![]
+            }
+        }
+    }
+}
+
+fn find_index(path: PathBuf) -> Option<PathBuf> {
+    let dir = std::fs::read_dir(path).ok()?;
+    dir.map(|e| e.unwrap())
+        .find(|e| e.path().file_stem().unwrap().eq_ignore_ascii_case("index"))
+        .map(|e| e.path())
+}
+
+#[derive(InputVisitable)]
+struct MakeReadNodes {
+    files: Input<Vec<PathBuf>>,
+    existing_nodes: HashMap<PathBuf, Input<ReadPostOutput>>,
+}
+impl MakeReadNodes {
+    fn new(files: Input<Vec<PathBuf>>) -> Self {
+        Self {
+            files,
+            existing_nodes: HashMap::new(),
+        }
+    }
+}
+impl DynamicRule for MakeReadNodes {
+    type ChildOutput = ReadPostOutput;
+    fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec<Input<Self::ChildOutput>> {
+        for file in self.files.value().iter() {
+            if !self.existing_nodes.contains_key(file) {
+                let input = ctx.add_rule(ReadPost { path: file.clone() });
+                self.existing_nodes.insert(file.clone(), input);
+            }
+        }
+        // collect everything up front so we can mutably borrow existing_nodes
+        let to_remove = self
+            .existing_nodes
+            .keys()
+            .filter(|key| !self.files.value().contains(key))
+            .cloned()
+            .collect::<Vec<_>>();
+        for key in to_remove {
+            ctx.remove_node(self.existing_nodes[&key].node_id());
+            self.existing_nodes.remove(&key);
+        }
+        self.existing_nodes.values().cloned().collect()
+    }
+}
+
+type ReadPostOutput = Option<Post<HtmlContent>>;
+
+#[derive(InputVisitable)]
+struct ReadPost {
+    path: PathBuf,
+}
+impl Rule for ReadPost {
+    type Output = ReadPostOutput;
+
+    fn evaluate(&mut self) -> Self::Output {
+        let buffer = std::fs::read_to_string(&self.path).ok()?;
+        match Post::new(self.path.clone(), &buffer) {
+            Ok(post) => Some(post.to_html()),
+            Err(e) => {
+                error!("Error parsing post {e:?}");
+                None
+            }
+        }
+    }
+
+    fn node_label(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            self.path
+                .components()
+                .last()
+                .unwrap()
+                .as_os_str()
+                .to_string_lossy()
+        )
+    }
+}
+
+#[derive(InputVisitable)]
+struct MakeExtractMetadatas {
+    posts: DynamicInput<ReadPostOutput>,
+    existing_nodes: HashMap<PathBuf, Input<ExtractMetadataOutput>>,
+}
+impl MakeExtractMetadatas {
+    fn new(posts: DynamicInput<ReadPostOutput>) -> Self {
+        Self {
+            posts,
+            existing_nodes: HashMap::new(),
+        }
+    }
+}
+impl DynamicRule for MakeExtractMetadatas {
+    type ChildOutput = ExtractMetadataOutput;
+    fn evaluate(&mut self, ctx: &mut impl DynamicRuleContext) -> Vec<Input<Self::ChildOutput>> {
+        let mut all_posts = vec![];
+        for post_input in self.posts.value().inputs.iter() {
+            let post_ = post_input.value();
+            let post = post_.as_ref().unwrap();
+            all_posts.push(post.path.clone());
+
+            if !self.existing_nodes.contains_key(&post.path) {
+                let input = ctx.add_rule(ExtractMetadata(post_input.clone()));
+                self.existing_nodes.insert(post.path.clone(), input);
+            }
+        }
+        let keys = self
+            .existing_nodes
+            .keys()
+            .filter(|key| !all_posts.contains(key))
+            .cloned()
+            .collect::<Vec<_>>();
+        for key in keys {
+            ctx.remove_node(self.existing_nodes[&key].node_id());
+            self.existing_nodes.remove(&key);
+        }
+        self.existing_nodes.values().cloned().collect()
+    }
+}
+
+type ExtractMetadataOutput = Option<PostMetadata>;
+
+#[derive(InputVisitable)]
+struct ExtractMetadata(Input<ReadPostOutput>);
+impl Rule for ExtractMetadata {
+    type Output = ExtractMetadataOutput;
+
+    fn evaluate(&mut self) -> Self::Output {
+        self.input_0().as_ref().map(|post| post.metadata.clone())
+    }
+
+    fn node_label(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if let Some(post) = self.input_0().as_ref() {
+            write!(f, "{}", post.slug)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Flattens Vec<Option<Post<HtmlContent>>> into Vec<Post<HtmlContent>>
+#[derive(InputVisitable)]
+struct AllPosts(DynamicInput<ReadPostOutput>);
+impl Rule for AllPosts {
+    type Output = Vec<Post<HtmlContent>>;
+    fn evaluate(&mut self) -> Self::Output {
+        self.input_0()
+            .inputs
+            .iter()
+            .flat_map(|inp| inp.value().clone())
+            .collect()
+    }
+}
+
+/// Flattens Vec<Option<PostMetadata>> into Vec<PostMetadata>
+#[derive(InputVisitable)]
+struct AllMetadatas(DynamicInput<ExtractMetadataOutput>);
+impl Rule for AllMetadatas {
+    type Output = Vec<PostMetadata>;
+    fn evaluate(&mut self) -> Self::Output {
+        self.input_0()
+            .inputs
+            .iter()
+            .flat_map(|inp| inp.value().clone())
+            .collect()
+    }
+}

src/generator/posts/content.rs

@@ -0,0 +1,125 @@
+use std::path::PathBuf;
+
+use anyhow::anyhow;
+
+use crate::generator::markdown;
+use crate::generator::posts::metadata::PostMetadata;
+use crate::generator::util::{from_frontmatter, slugify::slugify, word_count};
+
+pub trait PostContent: std::fmt::Debug + PartialEq {
+    fn to_html(self) -> HtmlContent;
+    fn word_count(&self) -> u32;
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct Post<Content: PostContent> {
+    pub path: PathBuf,
+    pub metadata: PostMetadata,
+    pub slug: String,
+    pub word_count: Option<u32>,
+    pub excerpt: Option<String>,
+    pub content: Content,
+}
+
+impl Post<AnyContent> {
+    pub fn new(path: PathBuf, contents: &str) -> anyhow::Result<Self> {
+        let (metadata, rest_contents) = match from_frontmatter::<PostMetadata>(contents) {
+            Ok(res) => res,
+            Err(e) => return Err(e),
+        };
+
+        let slug = metadata
+            .slug
+            .clone()
+            .unwrap_or_else(|| slugify(&metadata.title));
+
+        let ext = path.extension().unwrap().to_str().unwrap();
+        let content = content_from(ext, rest_contents)?;
+
+        Ok(Post {
+            path,
+            metadata,
+            slug: slug.to_owned(),
+            word_count: None,
+            excerpt: None,
+            content,
+        })
+    }
+}
+
+impl<C: PostContent> Post<C> {
+    pub fn to_html(self) -> Post<HtmlContent> {
+        Post {
+            path: self.path,
+            metadata: self.metadata,
+            slug: self.slug,
+            word_count: self.word_count,
+            excerpt: self.excerpt,
+            content: self.content.to_html(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum AnyContent {
+    Markdown(MarkdownContent),
+    Html(HtmlContent),
+}
+
+impl PostContent for AnyContent {
+    fn to_html(self) -> HtmlContent {
+        match self {
+            AnyContent::Markdown(inner) => inner.to_html(),
+            AnyContent::Html(inner) => inner,
+        }
+    }
+
+    fn word_count(&self) -> u32 {
+        match self {
+            AnyContent::Markdown(inner) => inner.word_count(),
+            AnyContent::Html(inner) => inner.word_count(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct MarkdownContent(String);
+
+impl PostContent for MarkdownContent {
+    fn to_html(self) -> HtmlContent {
+        let mut buf = vec![];
+        markdown::render(&self.0, &mut buf);
+        HtmlContent(String::from_utf8(buf).unwrap())
+    }
+
+    fn word_count(&self) -> u32 {
+        word_count::markdown(&self.0)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct HtmlContent(String);
+
+impl PostContent for HtmlContent {
+    fn to_html(self) -> HtmlContent {
+        self
+    }
+
+    fn word_count(&self) -> u32 {
+        word_count::html(&self.0)
+    }
+}
+
+impl HtmlContent {
+    pub fn html(&self) -> &str {
+        &self.0
+    }
+}
+
+fn content_from(extension: &str, content: &str) -> anyhow::Result<AnyContent> {
+    match extension {
+        "md" => Ok(AnyContent::Markdown(MarkdownContent(content.to_owned()))),
+        "html" => Ok(AnyContent::Html(HtmlContent(content.to_owned()))),
+        _ => Err(anyhow!("unknown extension {}", extension)),
+    }
+}

src/generator/posts/metadata.rs

@@ -0,0 +1,158 @@
+use std::hash::Hash;
+
+use chrono::{DateTime, FixedOffset};
+use serde::Deserialize;
+use serde::de::{SeqAccess, Visitor};
+
+use crate::generator::util::slugify::slugify;
+
+#[derive(Debug, Deserialize, PartialEq, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct PostMetadata {
+    pub title: String,
+    pub html_title: Option<String>,
+    pub tags: Option<Vec<Tag>>,
+    pub date: DateTime<FixedOffset>,
+    pub short_desc: Option<String>,
+    pub slug: Option<String>,
+    pub preamble: Option<String>,
+    #[serde(deserialize_with = "deserialize_old_permalink", default)]
+    pub old_permalink: Option<Vec<String>>,
+    pub use_old_permalink_for_comments: Option<bool>,
+    pub card_image_path: Option<String>,
+}
+
+fn deserialize_old_permalink<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    struct StringOrVec;
+    impl<'de> Visitor<'de> for StringOrVec {
+        type Value = Option<Vec<String>>;
+
+        fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+            formatter.write_str("string or vec of strings")
+        }
+
+        fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+        where
+            E: serde::de::Error,
+        {
+            Ok(Some(vec![value.to_owned()]))
+        }
+
+        fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+        where
+            A: SeqAccess<'de>,
+        {
+            let mut vec = Vec::with_capacity(seq.size_hint().unwrap_or(1));
+            loop {
+                match seq.next_element::<String>() {
+                    Ok(Some(s)) => vec.push(s),
+                    Ok(None) => break,
+                    Err(e) => return Err(e),
+                }
+            }
+            Ok(Some(vec))
+        }
+
+        fn visit_none<E>(self) -> Result<Self::Value, E>
+        where
+            E: serde::de::Error,
+        {
+            println!("visit_none");
+            Ok(None)
+        }
+    }
+
+    deserializer.deserialize_any(StringOrVec)
+}
+
+#[derive(Debug, Eq, Clone)]
+pub struct Tag {
+    pub name: String,
+    pub slug: String,
+}
+
+impl<'de> Deserialize<'de> for Tag {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let name = String::deserialize(deserializer)?;
+        let slug = slugify(&name);
+        Ok(Tag { name, slug })
+    }
+}
+
+impl PartialEq for Tag {
+    fn eq(&self, other: &Self) -> bool {
+        self.slug == other.slug
+    }
+}
+
+impl Hash for Tag {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.slug.hash(state);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use chrono::{DateTime, FixedOffset, TimeZone};
+    use serde::Deserialize;
+
+    #[derive(Deserialize)]
+    struct DeserializedDate {
+        d: DateTime<FixedOffset>,
+    }
+
+    #[test]
+    fn test_deserialize_date() {
+        let deserialized: DeserializedDate =
+            toml::from_str(r#"d = "2017-02-17 14:30:42 -0400""#).unwrap();
+        let expected = FixedOffset::west(4 * 3600)
+            .ymd(2017, 2, 17)
+            .and_hms(14, 30, 42);
+        assert_eq!(deserialized.d, expected);
+    }
+
+    #[test]
+    fn test_deserialize_old_permalink() {
+        let none: super::PostMetadata = toml::from_str(
+            r#"
+title = "Mocking HTTP Requests for iOS App UI Tests"
+date = "2019-12-22 19:12:42 -0400"
+            "#,
+        )
+        .unwrap();
+        assert_eq!(none.old_permalink, None);
+        let single: super::PostMetadata = toml::from_str(
+            r#"
+title = "Mocking HTTP Requests for iOS App UI Tests"
+date = "2019-12-22 19:12:42 -0400"
+old_permalink = "/ios/2019/mock-http-ios-ui-testing/"
+            "#,
+        )
+        .unwrap();
+        assert_eq!(
+            single.old_permalink,
+            Some(vec!["/ios/2019/mock-http-ios-ui-testing/".into()])
+        );
+        let multi: super::PostMetadata = toml::from_str(
+            r#"
+title = "Mocking HTTP Requests for iOS App UI Tests"
+date = "2019-12-22 19:12:42 -0400"
+old_permalink = ["/ios/2019/mock-http-ios-ui-testing/", "something else"]
+            "#,
+        )
+        .unwrap();
+        assert_eq!(
+            multi.old_permalink,
+            Some(vec![
+                "/ios/2019/mock-http-ios-ui-testing/".into(),
+                "something else".into()
+            ])
+        );
+    }
+}

src/generator/util/mod.rs

@@ -1,59 +1,14 @@
 pub mod one_more;
 pub mod slugify;
-pub mod templates;
 pub mod word_count;
 
+use std::path::{Component, Path, PathBuf};
+
 use anyhow::anyhow;
-use askama::Template;
-use serde::Deserialize;
-use std::fs::File;
-use std::io::{BufWriter, Write};
-use std::path::Path;
+use compute_graph::rule::{DynamicInput, Input, InputVisitable, Rule};
+use serde::de::DeserializeOwned;
 
-pub fn output_writer(path: impl AsRef<Path>) -> Result<impl Write, std::io::Error> {
-    let path = crate::generator::output_path(path);
-    if let Some(parent) = path.parent() {
-        std::fs::create_dir_all(parent)?;
-    }
-    let file = File::create(path)?;
-    Ok(BufWriter::new(file))
-}
-
-pub fn output_rendered_template(
-    template: &impl Template,
-    file: impl AsRef<Path>,
-) -> Result<(), std::io::Error> {
-    let path = file.as_ref();
-    let writer = output_writer(path)?;
-    template.render_into(&mut FmtWriter(writer)).map_err(|e| {
-        std::io::Error::new(
-            std::io::ErrorKind::Other,
-            format!("writing template {}: {}", path.display(), e),
-        )
-    })
-}
-
-struct FmtWriter<W: Write>(W);
-
-impl<W: Write> std::fmt::Write for FmtWriter<W> {
-    fn write_str(&mut self, s: &str) -> std::fmt::Result {
-        self.0.write_all(s.as_bytes()).map_err(|_| std::fmt::Error)
-    }
-
-    fn write_char(&mut self, c: char) -> std::fmt::Result {
-        let mut buf = [0u8; 4];
-        c.encode_utf8(&mut buf);
-        self.0.write_all(&buf).map_err(|_| std::fmt::Error)
-    }
-
-    fn write_fmt(&mut self, args: std::fmt::Arguments<'_>) -> std::fmt::Result {
-        self.0.write_fmt(args).map_err(|_| std::fmt::Error)
-    }
-}
-
-pub fn from_frontmatter<'de, D: Deserialize<'de>>(
-    contents: &'de str,
-) -> anyhow::Result<(D, &'de str)> {
+pub fn from_frontmatter<D: DeserializeOwned>(contents: &str) -> anyhow::Result<(D, &str)> {
     let mut chars = contents.char_indices();
     for i in 0..=2 {
         if chars.next() != Some((i, '`')) {
@@ -75,6 +30,59 @@ pub fn from_frontmatter<'de, D: Deserialize<'de>>(
     }
     let end_index = end_index.ok_or(anyhow!("missing frontmatter end"))?;
     let frontmatter = &contents[3..=end_index];
-    let deserialized = toml::from_str::<'de, D>(frontmatter)?;
+    let deserialized = toml::from_str(frontmatter)?;
     Ok((deserialized, chars.as_str()))
 }
+
+pub fn content_path(p: impl AsRef<Path>) -> PathBuf {
+    let mut buf = PathBuf::from("site_test/");
+    join_abs(&mut buf, p.as_ref());
+    buf
+}
+
+pub fn output_path(p: impl AsRef<Path>) -> PathBuf {
+    let mut buf = PathBuf::from("out/");
+    join_abs(&mut buf, p.as_ref());
+    buf
+}
+
+fn join_abs(buf: &mut PathBuf, p: &Path) {
+    for comp in p.components() {
+        match comp {
+            Component::RootDir => (),
+            Component::CurDir => (),
+            Component::Normal(c) => buf.push(c),
+            Component::Prefix(_) => panic!("prefixes are unsupported"),
+            Component::ParentDir => {
+                buf.pop();
+            }
+        }
+    }
+}
+
+#[derive(InputVisitable)]
+pub struct MapToVoid<T: 'static>(pub Input<T>);
+impl<T> Rule for MapToVoid<T> {
+    type Output = ();
+    fn evaluate(&mut self) -> Self::Output {}
+}
+
+#[derive(InputVisitable)]
+pub struct MapDynamicToVoid<T: 'static>(pub DynamicInput<T>);
+impl<T> Rule for MapDynamicToVoid<T> {
+    type Output = ();
+    fn evaluate(&mut self) -> Self::Output {}
+}
+
+#[cfg(test)]
+mod tests {
+    use super::join_abs;
+    use std::path::{Path, PathBuf};
+
+    #[test]
+    fn test_join_abs() {
+        let mut buf = PathBuf::from("site/");
+        join_abs(&mut buf, Path::new("/2022/test/"));
+        assert_eq!(buf, PathBuf::from("site/2022/test/"));
+    }
+}

src/main.rs

@@ -1,33 +1,9 @@
 #![feature(let_chains)]
 
-mod activitypub;
 mod generator;
-mod graph_generator;
 
-use crate::generator::{HtmlContent, Post};
-use axum::{
-    body::{boxed, Body, Bytes},
-    http::{Request, StatusCode},
-    response::{Html, IntoResponse, Redirect, Response},
-    routing::{get, get_service},
-    Extension, Router,
-};
-use clap::{arg, command, Command};
-use generator::output_path;
-use log::error;
+use clap::{Command, arg, command};
 use log::info;
-use notify_debouncer_mini::new_debouncer;
-use once_cell::sync::Lazy;
-use sqlx::{
-    sqlite::{SqliteConnectOptions, SqlitePoolOptions},
-    ConnectOptions, SqlitePool,
-};
-use std::path::Path;
-use std::{convert::Infallible, net::SocketAddr};
-use tokio::io::AsyncReadExt;
-use tokio_cron_scheduler::{Job, JobScheduler};
-use tower::Service;
-use tower_http::services::{ServeDir, ServeFile};
 
 #[tokio::main]
 async fn main() {
@@ -36,7 +12,6 @@ async fn main() {
     let matches = command!()
         .subcommand_required(true)
         .arg_required_else_help(true)
-        .subcommand(Command::new("graph-gen"))
         .subcommand(Command::new("gen"))
         .subcommand(
             Command::new("serve")
@@ -51,182 +26,12 @@ async fn main() {
     }
 
     match matches.subcommand() {
-        Some(("graph-gen", _)) => {
-            graph_generator::generate().await.unwrap();
-        }
         Some(("gen", _)) => {
-            let _ = generate().await;
+            generator::generate().await.expect("generating");
         }
-        Some(("serve", matches)) => {
-            // ensure that the keys are loadable
-            _ = Lazy::force(&activitypub::keys::PUB_KEY_PEM);
-            _ = Lazy::force(&activitypub::keys::PRIV_KEY_PEM);
-
-            let posts = generate().await.expect("initial generation");
-            info!("Generated");
-
-            let pool = setup_db().await;
-            activitypub::articles::insert_posts(&posts, &pool).await;
-
-            if matches.is_present("watch") {
-                start_watcher();
-            }
-
-            let pool_ = pool.clone();
-            tokio::spawn(async move {
-                match activitypub::articles::federate_outgoing(&pool_).await {
-                    Ok(()) => (),
-                    Err(e) => error!("Federating outgoing articles: {:?}", e),
-                }
-            });
-
-            let sched = JobScheduler::new().await.expect("create JobScheduler");
-            let digest_schedule = if cfg!(debug_assertions) {
-                // every 5 minutes in debug
-                "0 1/5 * * * *"
-            } else {
-                // every day at midnight utc
-                "0 0 0 * * * *"
-            };
-            let pool_ = pool.clone();
-            sched
-                .add(
-                    Job::new_async(digest_schedule, move |_, _| {
-                        // this closure executes multiple times, so we need to clone the pool every
-                        // time rather than moving it into the closure
-                        let pool = pool_.clone();
-                        Box::pin(async move {
-                            activitypub::digester::send_digest_if_necessary(&pool).await;
-                        })
-                    })
-                    .expect("creating digest job"),
-                )
-                .await
-                .expect("adding digest job");
-            sched.start().await.expect("starting JobScheduler");
-
-            serve(&posts, pool).await;
+        Some(("serve", _matches)) => {
+            todo!()
         }
         _ => unreachable!(),
     }
 }
-
-async fn generate() -> anyhow::Result<Vec<Post<HtmlContent>>> {
-    generator::generate().await
-}
-
-fn start_watcher() {
-    let handle = tokio::runtime::Handle::current();
-
-    std::thread::spawn(move || {
-        let (tx, rx) = std::sync::mpsc::channel();
-
-        let mut debouncer =
-            new_debouncer(std::time::Duration::from_millis(100), None, tx).expect("debouncer");
-
-        debouncer
-            .watcher()
-            .watch(Path::new("site/"), notify::RecursiveMode::Recursive)
-            .expect("watch");
-
-        info!("Started watcher");
-
-        for events in rx {
-            let events = events.unwrap();
-            let paths = events.iter().map(|ev| &ev.path).collect::<Vec<_>>();
-            info!("Regenerating due to changes at {:?}", paths);
-            handle.spawn(async move {
-                match generate().await {
-                    Ok(_) => (),
-                    Err(e) => error!("Regeneration failed: {:?}", e),
-                }
-            });
-        }
-    });
-}
-
-async fn setup_db() -> SqlitePool {
-    let mut options = SqliteConnectOptions::new()
-        .filename(std::env::var("DB_PATH").unwrap_or("db.sqlite".to_owned()));
-    options.log_statements(log::LevelFilter::Debug);
-    let pool = SqlitePoolOptions::new()
-        .connect_with(options)
-        .await
-        .unwrap();
-
-    sqlx::migrate!()
-        .run(&pool)
-        .await
-        .expect("database migrated");
-
-    pool
-}
-
-async fn serve(posts: &[Post<HtmlContent>], pool: SqlitePool) {
-    let articles_or_fallback = tower::service_fn(handle_ap_article_or_serve_dir);
-
-    let app = Router::new()
-        .merge(activitypub::router(pool.clone()))
-        .merge(redirect_router(posts))
-        .fallback(get_service(articles_or_fallback))
-        .layer(Extension(pool));
-
-    let addr = if cfg!(debug_assertions) {
-        SocketAddr::from(([0, 0, 0, 0], 8084))
-    } else {
-        SocketAddr::from(([127, 0, 0, 1], 8084))
-    };
-    info!("Listening on {}", addr);
-    axum::Server::bind(&addr)
-        .serve(app.into_make_service())
-        .await
-        .unwrap();
-}
-
-async fn handle_ap_article_or_serve_dir(req: Request<Body>) -> Result<Response, Infallible> {
-    let articles_res = activitypub::articles::handle(&req).await;
-
-    match articles_res {
-        Some(res) => Ok(res.into_response()),
-        None => {
-            let mut serve_dir =
-                ServeDir::new("out").fallback(ServeFile::new(output_path("404.html")));
-            match serve_dir.call(req).await {
-                Ok(resp) => Ok(resp.map(boxed)),
-                Err(e) => Ok(handle_error(e).await.into_response()),
-            }
-        }
-    }
-}
-
-fn redirect_router(posts: &[Post<HtmlContent>]) -> Router {
-    let mut r = Router::new();
-    for post in posts.iter() {
-        for path in post.metadata.old_permalink.iter().flatten() {
-            let new_permalink = post.permalink();
-            r = r.route(
-                path,
-                get(|_: Request<Body>| async move { Redirect::permanent(&new_permalink) }),
-            );
-        }
-    }
-    r
-}
-
-async fn handle_error(err: std::io::Error) -> impl IntoResponse {
-    error!("Unhandled error: {}", err);
-    (
-        StatusCode::INTERNAL_SERVER_ERROR,
-        serve_500().await.unwrap_or_else(|err| {
-            error!("Unhandled error serving 500 page: {}", &err);
-            "Internal server error".into_response()
-        }),
-    )
-}
-
-async fn serve_500() -> Result<Response, std::io::Error> {
-    let mut buf = vec![];
-    let mut f = tokio::fs::File::open(output_path("500.html")).await?;
-    f.read_to_end(&mut buf).await?;
-    Ok(Html(Bytes::from(buf)).into_response())
-}