diff --git a/tools/automator/src/pull_requests.rs b/tools/automator/src/pull_requests.rs
index 102174632..fa8b08a1a 100644
--- a/tools/automator/src/pull_requests.rs
+++ b/tools/automator/src/pull_requests.rs
@@ -20,7 +20,7 @@ impl PullRequestsSinceLastRelease {
         let mut pull_requests = BTreeMap::new();
         let mut page = 1u32;
 
-        let version_of_last_release = 'outer: loop {
+        let (version_of_last_release, time_of_last_release) = 'outer: loop {
             const MAX_RESULTS_PER_PAGE: u8 = 100;
 
             println!("Fetching page {}...", page);
@@ -60,7 +60,12 @@ impl PullRequestsSinceLastRelease {
                                 )
                             })?;
 
-                            break 'outer version;
+                            let time =
+                                pull_request.merged_at.ok_or_else(|| {
+                                    anyhow!("Release PR is missing merge time")
+                                })?;
+
+                            break 'outer (version, time);
                         }
                     }
                 }
@@ -99,6 +104,10 @@ impl PullRequestsSinceLastRelease {
             }
         };
 
+        pull_requests.retain(|_, pull_request| {
+            pull_request.merged_at > time_of_last_release
+        });
+
         Ok(Self {
             pull_requests,
             version_of_last_release,