diff --git a/crates/egui/src/widget_rect.rs b/crates/egui/src/widget_rect.rs
index c39b28b50..e03423623 100644
--- a/crates/egui/src/widget_rect.rs
+++ b/crates/egui/src/widget_rect.rs
@@ -178,14 +178,6 @@ impl WidgetRects {
                 // e.g. calling `response.interact(…)` to add more interaction.
                 let (idx_in_layer, existing) = entry.get_mut();
 
-                debug_assert!(
-                    existing.layer_id == widget_rect.layer_id,
-                    "Widget {:?} changed layer_id during the frame from {:?} to {:?}",
-                    widget_rect.id,
-                    existing.layer_id,
-                    widget_rect.layer_id
-                );
-
                 // Update it:
                 existing.rect = widget_rect.rect; // last wins
                 existing.interact_rect = widget_rect.interact_rect; // last wins
@@ -200,6 +192,11 @@ impl WidgetRects {
                     } else {
                         layer_widgets[*idx_in_layer] = *existing;
                     }
+                } else if cfg!(debug_assertions) {
+                    panic!(
+                        "DEBUG ASSERT: Widget {:?} changed layer_id during the frame from {:?} to {:?}",
+                        widget_rect.id, existing.layer_id, widget_rect.layer_id
+                    );
                 }
             }
         }