diff --git a/src/kaleidoscope/driver/hid/base/Keyboard.h b/src/kaleidoscope/driver/hid/base/Keyboard.h
index b553d42d..8ebeb894 100644
--- a/src/kaleidoscope/driver/hid/base/Keyboard.h
+++ b/src/kaleidoscope/driver/hid/base/Keyboard.h
@@ -306,7 +306,7 @@ class Keyboard {
   void releaseKey(Key released_key) {
     // Remove any modifiers attached to this key from the bitmask of modifiers we're
     // willing to attach to USB HID keyboard reports
-    modifier_flag_mask ^= released_key.getFlags();
+    modifier_flag_mask &= ~(released_key.getFlags());
 
     if (!isModifierKey(released_key)) {
 
@@ -445,7 +445,7 @@ class Keyboard {
   // to the next USB HID report and removes them from the bitmap of all such modifiers.
 
   void cancelModifierRequest(byte flags) {
-    requested_modifier_flags ^= flags;
+    requested_modifier_flags &= ~flags;
   }
 
   // pressModifiers takes a bitmap of modifier keys that must be included in