diff --git a/Cargo.lock b/Cargo.lock index 293d5813..fc1cfce7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -772,6 +772,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -1153,6 +1162,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "file-id" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6584280525fb2059cba3db2c04abf947a1a29a45ddae89f3870f8281704fafc9" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "filetime" version = "0.2.23" @@ -1214,21 +1232,11 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fsevent" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab7d1bd1bd33cc98b0889831b72da23c0aa4df9cec7e0702f46ecea04b35db6" -dependencies = [ - "bitflags 1.3.2", - "fsevent-sys", -] - [[package]] name = "fsevent-sys" -version = "2.0.1" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41b048a94555da0f42f1d632e2e19510084fb8e303b0daa2816e733fb3644a0" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" dependencies = [ "libc", ] @@ -1754,9 +1762,9 @@ dependencies = [ [[package]] name = "inotify" -version = "0.7.1" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4816c66d2c8ae673df83366c18341538f234a26d65a9ecea5c348b453ac1d02f" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" dependencies = [ "bitflags 1.3.2", "inotify-sys", @@ -1917,6 +1925,26 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lasso" version = "0.7.2" @@ -2449,6 +2477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -2610,20 +2639,35 @@ checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" [[package]] name = "notify" -version = "4.0.17" +version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae03c8c853dba7bfd23e571ff0cff7bc9dceb40a4cd684cd1681824183f45257" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", + "crossbeam-channel", "filetime", - "fsevent", "fsevent-sys", "inotify", + "kqueue", "libc", - "mio 0.6.23", - "mio-extras", + "log", + "mio 0.8.11", + "walkdir", + "windows-sys 0.48.0", +] + +[[package]] +name = "notify-debouncer-full" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f5dab59c348b9b50cf7f261960a20e389feb2713636399cd9082cd4b536154" +dependencies = [ + "crossbeam-channel", + "file-id", + "log", + "notify", + "parking_lot", "walkdir", - "winapi 0.3.9", ] [[package]] @@ -5076,7 +5120,7 @@ dependencies = [ "libs", "mime", "mime_guess", - "notify", + "notify-debouncer-full", "open", "pathdiff", "same-file", diff --git a/Cargo.toml b/Cargo.toml index 1db7548d..722dbe5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ clap_complete = "4" hyper = { version = "0.14.1", default-features = false, features = ["runtime", "server", "http2", "http1"] } tokio = { version = "1.0.1", default-features = false, features = ["rt", "fs", "time"] } time = { version = "0.3", features = ["formatting", "macros", "local-offset"] } -notify = "4" +notify-debouncer-full = "0.3" ws = "0.9" ctrlc = "3" open = "5" diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index b9068851..a84ca526 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -52,7 +52,9 @@ pub struct Site { pub live_reload: Option, pub output_path: PathBuf, content_path: PathBuf, + pub sass_path: PathBuf, pub static_path: PathBuf, + pub templates_path: PathBuf, pub taxonomies: Vec, /// A map of all .md files (section and pages) and their permalink /// We need that if there are relative links in the content that need to be resolved @@ -82,7 +84,9 @@ impl Site { let shortcode_definitions = utils::templates::get_shortcodes(&tera); let content_path = path.join("content"); + let sass_path = path.join("sass"); let static_path = path.join("static"); + let templates_path = path.join("templates"); let imageproc = imageproc::Processor::new(path.to_path_buf(), &config); let output_path = path.join(config.output_dir.clone()); @@ -94,7 +98,9 @@ impl Site { live_reload: None, output_path, content_path, + sass_path, static_path, + templates_path, taxonomies: Vec::new(), permalinks: HashMap::new(), include_drafts: false, diff --git a/src/cmd/serve.rs b/src/cmd/serve.rs index d3aa01f6..392ca73a 100644 --- a/src/cmd/serve.rs +++ b/src/cmd/serve.rs @@ -22,6 +22,7 @@ // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. use std::cell::Cell; +use std::collections::HashMap; use std::fs::read_dir; use std::future::IntoFuture; use std::net::{IpAddr, SocketAddr, TcpListener}; @@ -44,7 +45,7 @@ use libs::globset::GlobSet; use libs::percent_encoding; use libs::relative_path::{RelativePath, RelativePathBuf}; use libs::serde_json; -use notify::{watcher, RecursiveMode, Watcher}; +use notify_debouncer_full::{new_debouncer, notify::RecursiveMode, notify::Watcher}; use ws::{Message, Sender, WebSocket}; use errors::{anyhow, Context, Error, Result}; @@ -53,10 +54,11 @@ use site::sass::compile_sass; use site::{Site, SITE_CONTENT}; use utils::fs::{clean_site_output_folder, copy_file, is_temp_file}; +use crate::fs_event_utils::{get_relevant_event_kind, SimpleFSEventKind}; use crate::messages; use std::ffi::OsStr; -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, Hash, PartialEq)] enum ChangeKind { Content, Templates, @@ -65,6 +67,7 @@ enum ChangeKind { Sass, Config, } +impl Eq for ChangeKind {} #[derive(Debug, PartialEq)] enum WatchMode { @@ -485,7 +488,7 @@ pub fn serve( // Setup watchers let (tx, rx) = channel(); - let mut watcher = watcher(tx, Duration::from_secs(1)).unwrap(); + let mut debouncer = new_debouncer(Duration::from_secs(1), /*tick_rate=*/ None, tx).unwrap(); // We watch for changes on the filesystem for every entry in watch_this // Will fail if either: @@ -501,8 +504,8 @@ pub fn serve( WatchMode::Condition(b) => b && watch_path.exists(), }; if should_watch { - watcher - .watch(root_dir.join(entry), RecursiveMode::Recursive) + debouncer.watcher() + .watch(&root_dir.join(entry), RecursiveMode::Recursive) .with_context(|| format!("Can't watch `{}` for changes in folder `{}`. Does it exist, and do you have correct permissions?", entry, root_dir.display()))?; watchers.push(entry.to_string()); } @@ -606,24 +609,24 @@ pub fn serve( }) .expect("Error setting Ctrl-C handler"); - use notify::DebouncedEvent::*; - - let reload_sass = |site: &Site, path: &Path, partial_path: &Path| { - let msg = if path.is_dir() { - format!("-> Directory in `sass` folder changed {}", path.display()) - } else { - format!("-> Sass file changed {}", path.display()) - }; + let reload_sass = |site: &Site, paths: &Vec<&PathBuf>| { + let combined_paths = + paths.iter().map(|p| p.display().to_string()).collect::>().join(", "); + let msg = format!("-> Sass file(s) changed {}", combined_paths); console::info(&msg); rebuild_done_handling( &broadcaster, compile_sass(&site.base_path, &site.output_path), - &partial_path.to_string_lossy(), + &site.sass_path.to_string_lossy(), ); }; - let reload_templates = |site: &mut Site, path: &Path| { - rebuild_done_handling(&broadcaster, site.reload_templates(), &path.to_string_lossy()); + let reload_templates = |site: &mut Site| { + rebuild_done_handling( + &broadcaster, + site.reload_templates(), + &site.templates_path.to_string_lossy(), + ); }; let copy_static = |site: &Site, path: &Path, partial_path: &Path| { @@ -690,52 +693,99 @@ pub fn serve( loop { match rx.recv() { - Ok(event) => { - let can_do_fast_reload = !matches!(event, Remove(_)); + Ok(Ok(mut events)) => { + // Arrange events from oldest to newest. + events.sort_by(|e1, e2| e1.time.cmp(&e2.time)); - match event { - // Intellij does weird things on edit, chmod is there to count those changes - // https://github.com/passcod/notify/issues/150#issuecomment-494912080 - Rename(_, path) | Create(path) | Write(path) | Remove(path) | Chmod(path) => { - if is_ignored_file(&site.config.ignored_content_globset, &path) { - continue; - } + // Use a map to keep only the last event that occurred for a particular path. + // Map `full_path -> (partial_path, simple_event_kind, change_kind)`. + let mut meaningful_events: HashMap< + PathBuf, + (PathBuf, SimpleFSEventKind, ChangeKind), + > = HashMap::new(); - if is_temp_file(&path) { - continue; - } + for event in events.iter() { + let simple_kind = get_relevant_event_kind(&event.event.kind); + if simple_kind.is_none() { + continue; + } - // We only care about changes in non-empty folders - if path.is_dir() && is_folder_empty(&path) { - continue; - } + // We currently only handle notify events that report a single path per event. + if event.event.paths.len() != 1 { + console::error(&format!( + "Skipping unsupported file system event with multiple paths: {:?}", + event.event.kind + )); + continue; + } + let path = event.event.paths[0].clone(); - let format = - format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"); - let current_time = - OffsetDateTime::now_utc().to_offset(utc_offset).format(&format); - if let Ok(time_str) = current_time { - println!("Change detected @ {}", time_str); - } else { - // if formatting fails for some reason - println!("Change detected"); - }; + if is_ignored_file(&site.config.ignored_content_globset, &path) { + continue; + } - let start = Instant::now(); - match detect_change_kind(root_dir, &path, &config_path) { - (ChangeKind::Content, _) => { - console::info(&format!("-> Content changed {}", path.display())); + if is_temp_file(&path) { + continue; + } + + // We only care about changes in non-empty folders + if path.is_dir() && is_folder_empty(&path) { + continue; + } + + let (change_k, partial_p) = detect_change_kind(&root_dir, &path, &config_path); + meaningful_events.insert(path, (partial_p, simple_kind.unwrap(), change_k)); + } + + if meaningful_events.is_empty() { + continue; + } + + // Bin changes by change kind to support later iteration over batches of changes. + // Map of change_kind -> (partial_path, full_path, event_kind). + let mut changes: HashMap< + ChangeKind, + Vec<(&PathBuf, &PathBuf, &SimpleFSEventKind)>, + > = HashMap::new(); + for (full_path, (partial_path, event_kind, change_kind)) in meaningful_events.iter() + { + let c = changes.entry(*change_kind).or_insert(vec![]); + c.push((partial_path, full_path, event_kind)); + } + + let format = format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"); + + for (change_kind, change_group) in changes.iter() { + let current_time = + OffsetDateTime::now_utc().to_offset(utc_offset).format(&format); + if let Ok(time_str) = current_time { + println!("Change detected @ {}", time_str); + } else { + // if formatting fails for some reason + println!("Change detected"); + }; + + let start = Instant::now(); + match change_kind { + ChangeKind::Content => { + for (_, full_path, event_kind) in change_group.iter() { + console::info(&format!( + "-> Content changed {}", + full_path.display() + )); + + let can_do_fast_reload = **event_kind != SimpleFSEventKind::Remove; if fast_rebuild { if can_do_fast_reload { - let filename = path + let filename = full_path .file_name() .unwrap_or_else(|| OsStr::new("")) .to_string_lossy(); let res = if filename == "_index.md" { - site.add_and_render_section(&path) + site.add_and_render_section(&full_path) } else if filename.ends_with(".md") { - site.add_and_render_page(&path) + site.add_and_render_page(&full_path) } else { // an asset changed? a folder renamed? // should we make it smarter so it doesn't reload the whole site? @@ -750,7 +800,7 @@ pub fn serve( rebuild_done_handling( &broadcaster, res, - &path.to_string_lossy(), + &full_path.to_string_lossy(), ); } } else { @@ -763,51 +813,64 @@ pub fn serve( site = s; } } - (ChangeKind::Templates, partial_path) => { - let msg = if path.is_dir() { - format!( - "-> Directory in `templates` folder changed {}", - path.display() - ) - } else { - format!("-> Template changed {}", path.display()) - }; - console::info(&msg); - - // A shortcode changed, we need to rebuild everything - if partial_path.starts_with("/templates/shortcodes") { - if let Some(s) = recreate_site() { - site = s; - } - } else { - println!("Reloading only template"); - // A normal template changed, no need to re-render Markdown. - reload_templates(&mut site, &path) - } - } - (ChangeKind::StaticFiles, p) => copy_static(&site, &path, &p), - (ChangeKind::Sass, p) => reload_sass(&site, &path, &p), - (ChangeKind::Themes, _) => { - console::info("-> Themes changed."); + } + ChangeKind::Templates => { + let partial_paths: Vec<&PathBuf> = + change_group.iter().map(|(p, _, _)| *p).collect(); + let full_paths: Vec<&PathBuf> = + change_group.iter().map(|(_, p, _)| *p).collect(); + let combined_paths = full_paths + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(", "); + let msg = format!("-> Template file(s) changed {}", combined_paths); + console::info(&msg); + let shortcodes_updated = partial_paths + .iter() + .any(|p| p.starts_with("/templates/shortcodes")); + // Rebuild site if shortcodes change; otherwise, just update template. + if shortcodes_updated { if let Some(s) = recreate_site() { site = s; } + } else { + println!("Reloading only template"); + reload_templates(&mut site) } - (ChangeKind::Config, _) => { - console::info("-> Config changed. The browser needs to be refreshed to make the changes visible."); + } + ChangeKind::StaticFiles => { + for (partial_path, full_path, _) in change_group.iter() { + copy_static(&site, &full_path, &partial_path); + } + } + ChangeKind::Sass => { + let full_paths = change_group.iter().map(|(_, p, _)| *p).collect(); + reload_sass(&site, &full_paths); + } + ChangeKind::Themes => { + // No need to iterate over change group since we're rebuilding the site. + console::info("-> Themes changed."); - if let Some(s) = recreate_site() { - site = s; - } + if let Some(s) = recreate_site() { + site = s; } - }; - messages::report_elapsed_time(start); - } - _ => {} + } + ChangeKind::Config => { + // No need to iterate over change group since we're rebuilding the site. + console::info("-> Config changed. The browser needs to be refreshed to make the changes visible."); + + if let Some(s) = recreate_site() { + site = s; + } + } + }; + messages::report_elapsed_time(start); } } - Err(e) => console::error(&format!("Watch error: {:?}", e)), + Ok(Err(e)) => console::error(&format!("File system event errors: {:?}", e)), + Err(e) => console::error(&format!("File system event receiver errors: {:?}", e)), }; } } diff --git a/src/fs_event_utils.rs b/src/fs_event_utils.rs new file mode 100644 index 00000000..7c2479f9 --- /dev/null +++ b/src/fs_event_utils.rs @@ -0,0 +1,80 @@ +//! Utilities to simplify working with events raised by the `notify*` family of file system +//! event-watching libraries. + +use notify_debouncer_full::notify::event::*; + +/// This enum abstracts over the fine-grained group of enums in `notify`. +#[derive(Clone, Debug, PartialEq)] +pub enum SimpleFSEventKind { + Create, + Modify, + Remove, +} + +/// Filter `notify_debouncer_full` events. For events that we care about, +/// return our internal simplified representation. For events we don't care about, +/// return `None`. +pub fn get_relevant_event_kind(event_kind: &EventKind) -> Option { + match event_kind { + EventKind::Create(CreateKind::File) | EventKind::Create(CreateKind::Folder) => { + Some(SimpleFSEventKind::Create) + } + EventKind::Modify(ModifyKind::Data(DataChange::Size)) + | EventKind::Modify(ModifyKind::Data(DataChange::Content)) + // Intellij modifies file metadata on edit. + // https://github.com/passcod/notify/issues/150#issuecomment-494912080 + | EventKind::Modify(ModifyKind::Metadata(MetadataKind::WriteTime)) + | EventKind::Modify(ModifyKind::Metadata(MetadataKind::Permissions)) + | EventKind::Modify(ModifyKind::Metadata(MetadataKind::Ownership)) + | EventKind::Modify(ModifyKind::Name(RenameMode::To)) => Some(SimpleFSEventKind::Modify), + EventKind::Remove(RemoveKind::File) | EventKind::Remove(RemoveKind::Folder) => { + Some(SimpleFSEventKind::Remove) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use notify_debouncer_full::notify::event::*; + + use super::{get_relevant_event_kind, SimpleFSEventKind}; + + // This test makes sure we at least have code coverage on the `notify` event kinds we care + // about when watching the file system for site changes. This is to make sure changes to the + // event mapping and filtering don't cause us to accidentally ignore things we care about. + #[test] + fn test_get_relative_event_kind() { + let cases = vec![ + (EventKind::Create(CreateKind::File), Some(SimpleFSEventKind::Create)), + (EventKind::Create(CreateKind::Folder), Some(SimpleFSEventKind::Create)), + ( + EventKind::Modify(ModifyKind::Data(DataChange::Size)), + Some(SimpleFSEventKind::Modify), + ), + ( + EventKind::Modify(ModifyKind::Data(DataChange::Content)), + Some(SimpleFSEventKind::Modify), + ), + ( + EventKind::Modify(ModifyKind::Metadata(MetadataKind::WriteTime)), + Some(SimpleFSEventKind::Modify), + ), + ( + EventKind::Modify(ModifyKind::Metadata(MetadataKind::Permissions)), + Some(SimpleFSEventKind::Modify), + ), + ( + EventKind::Modify(ModifyKind::Metadata(MetadataKind::Ownership)), + Some(SimpleFSEventKind::Modify), + ), + (EventKind::Modify(ModifyKind::Name(RenameMode::To)), Some(SimpleFSEventKind::Modify)), + (EventKind::Remove(RemoveKind::File), Some(SimpleFSEventKind::Remove)), + (EventKind::Remove(RemoveKind::Folder), Some(SimpleFSEventKind::Remove)), + ]; + for (case, expected) in cases.iter() { + let ek = get_relevant_event_kind(&case); + assert_eq!(ek, *expected); + } + } +} diff --git a/src/main.rs b/src/main.rs index 48e95b96..6cbcd53f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use time::UtcOffset; mod cli; mod cmd; +mod fs_event_utils; mod messages; mod prompt;