diff --git a/src/town_gui.cpp b/src/town_gui.cpp
index ea80af3c57..9076b0b56d 100644
--- a/src/town_gui.cpp
+++ b/src/town_gui.cpp
@@ -297,7 +297,7 @@ public:
 				}
 
 				/* When double-clicking, continue */
-				if (click_count == 1 || y < 0) break;
+				if (click_count == 1 || y < 0 || !HasBit(this->available_actions, y)) break;
 				FALLTHROUGH;
 			}