diff --git a/src/db.rs b/src/db.rs
index a02d105..b98d675 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -177,8 +177,6 @@ impl Db {
                 .map(|expiration| expiration > when)
                 .unwrap_or(true);
 
-            // Track the expiration.
-            state.expirations.insert((when, key.clone()));
             when
         });
 
@@ -197,10 +195,17 @@ impl Db {
         if let Some(prev) = prev {
             if let Some(when) = prev.expires_at {
                 // clear expiration
-                state.expirations.remove(&(when, key));
+                state.expirations.remove(&(when, key.clone()));
             }
         }
 
+        // Track the expiration. If we insert before remove that will cause bug
+        // when current `(when, key)` equals prev `(when, key)`. Remove then insert
+        // can avoid this.
+        if let Some(when) = expires_at {
+            state.expirations.insert((when, key));
+        }
+
         // Release the mutex before notifying the background task. This helps
         // reduce contention by avoiding the background task waking up only to
         // be unable to acquire the mutex due to this function still holding it.