diff --git a/src/viewport.cpp b/src/viewport.cpp
index 5d3fcab24a..66477c223f 100644
--- a/src/viewport.cpp
+++ b/src/viewport.cpp
@@ -2848,27 +2848,57 @@ void UpdateViewportPosition(Window *w)
 
 		ClampViewportToMap(vp, w->viewport->scrollpos_x, w->viewport->scrollpos_y);
 
-		if (_scrolling_viewport == w && _settings_client.gui.show_scrolling_viewport_on_map) {
-			const int gap = ScaleByZoom(1, ZOOM_LVL_MAX);
-
-			int lr_low = vp->virtual_left;
-			int lr_hi = w->viewport->scrollpos_x;
-			if (lr_low > lr_hi) Swap(lr_low, lr_hi);
-			int right = lr_hi + vp->virtual_width + gap;
-
-			int tb_low = vp->virtual_top;
-			int tb_hi = w->viewport->scrollpos_y;
-			if (tb_low > tb_hi) Swap(tb_low, tb_hi);
-			int bottom = tb_hi + vp->virtual_height + gap;
-
-			MarkAllViewportMapsDirty(lr_low, tb_low, right, bottom);
-		}
+		if (_scrolling_viewport == w) UpdateActiveScrollingViewport(w);
 
 		SetViewportPosition(w, w->viewport->scrollpos_x, w->viewport->scrollpos_y);
 		if (update_overlay) RebuildViewportOverlay(w);
 	}
 }
 
+void UpdateActiveScrollingViewport(Window *w)
+{
+	if (w && (!_settings_client.gui.show_scrolling_viewport_on_map || w->viewport->zoom >= ZOOM_LVL_DRAW_MAP)) w = nullptr;
+
+	const bool bound_valid = (_scrolling_viewport_bound.left != _scrolling_viewport_bound.right);
+
+	if (!w && !bound_valid) return;
+
+	const int gap = ScaleByZoom(1, ZOOM_LVL_MAX);
+
+	auto get_bounds = [&gap](const ViewportData *vp) -> Rect {
+		int lr_low = vp->virtual_left;
+		int lr_hi = vp->dest_scrollpos_x;
+		if (lr_low > lr_hi) Swap(lr_low, lr_hi);
+		int right = lr_hi + vp->virtual_width + gap;
+
+		int tb_low = vp->virtual_top;
+		int tb_hi = vp->scrollpos_y;
+		if (tb_low > tb_hi) Swap(tb_low, tb_hi);
+		int bottom = tb_hi + vp->virtual_height + gap;
+
+		return { lr_low, tb_low, right, bottom };
+	};
+
+	if (w && !bound_valid) {
+		const Rect bounds = get_bounds(w->viewport);
+		MarkAllViewportMapsDirty(bounds.left, bounds.top, bounds.right, bounds.bottom);
+		_scrolling_viewport_bound = bounds;
+	} else if (!w && bound_valid) {
+		const Rect &bounds = _scrolling_viewport_bound;
+		MarkAllViewportMapsDirty(bounds.left, bounds.top, bounds.right, bounds.bottom);
+		_scrolling_viewport_bound = { 0, 0, 0, 0 };
+	} else {
+		/* Calculate symmetric difference of two rectangles */
+		const Rect a = get_bounds(w->viewport);
+		const Rect &b = _scrolling_viewport_bound;
+		if (a.left != b.left) MarkAllViewportMapsDirty(min(a.left, b.left) - gap, min(a.top, b.top) - gap, max(a.left, b.left) + gap, max(a.bottom, b.bottom) + gap);
+		if (a.top != b.top) MarkAllViewportMapsDirty(min(a.left, b.left) - gap, min(a.top, b.top) - gap, max(a.right, b.right) + gap, max(a.top, b.top) + gap);
+		if (a.right != b.right) MarkAllViewportMapsDirty(min(a.right, b.right) - (2 * gap), min(a.top, b.top) - gap, max(a.right, b.right) + gap, max(a.bottom, b.bottom) + gap);
+		if (a.bottom != b.bottom) MarkAllViewportMapsDirty(min(a.left, b.left) - gap, min(a.bottom, b.bottom) - (2 * gap), max(a.right, b.right) + gap, max(a.bottom, b.bottom) + gap);
+		_scrolling_viewport_bound = a;
+	}
+}
+
 /**
  * Marks a viewport as dirty for repaint if it displays (a part of) the area the needs to be repainted.
  * @param vp     The viewport to mark as dirty
@@ -2886,9 +2916,11 @@ static void MarkViewportDirty(const ViewPort * const vp, int left, int top, int
 
 	right -= vp->virtual_left;
 	if (right <= 0) return;
+	right = min(right, vp->virtual_width);
 
 	bottom -= vp->virtual_top;
 	if (bottom <= 0) return;
+	bottom = min(bottom, vp->virtual_height);
 
 	left = max(0, left - vp->virtual_left);
 
diff --git a/src/viewport_func.h b/src/viewport_func.h
index 5aa40daa7d..d7d0159e3f 100644
--- a/src/viewport_func.h
+++ b/src/viewport_func.h
@@ -78,6 +78,8 @@ void ViewportDoDraw(const ViewPort *vp, int left, int top, int right, int bottom
 bool ScrollWindowToTile(TileIndex tile, Window *w, bool instant = false);
 bool ScrollWindowTo(int x, int y, int z, Window *w, bool instant = false);
 
+void UpdateActiveScrollingViewport(Window *w);
+
 void RebuildViewportOverlay(Window *w);
 
 bool ScrollMainWindowToTile(TileIndex tile, bool instant = false);
diff --git a/src/window.cpp b/src/window.cpp
index b366fc2fd6..1ca3153e98 100644
--- a/src/window.cpp
+++ b/src/window.cpp
@@ -75,6 +75,7 @@ int _scrollbar_size;
 byte _scroller_click_timeout = 0;
 
 Window *_scrolling_viewport; ///< A viewport is being scrolled with the mouse.
+Rect _scrolling_viewport_bound; ///< A viewport is being scrolled with the mouse, the overlay currently covers this viewport rectangle.
 bool _mouse_hovering;      ///< The mouse is hovering over the same point.
 
 SpecialMouseMode _special_mouse_mode; ///< Mode of the mouse.
@@ -1880,6 +1881,7 @@ void InitWindowSystem()
 	_mouseover_last_w = NULL;
 	_last_scroll_window = NULL;
 	_scrolling_viewport = NULL;
+	_scrolling_viewport_bound = { 0, 0, 0, 0 };
 	_mouse_hovering = false;
 
 	NWidgetLeaf::InvalidateDimensionCache(); // Reset cached sizes of several widgets.
@@ -2450,6 +2452,7 @@ static EventState HandleViewportScroll()
 		_cursor.fix_at = false;
 		_scrolling_viewport = NULL;
 		_last_scroll_window = NULL;
+		UpdateActiveScrollingViewport(nullptr);
 		return ES_NOT_HANDLED;
 	}
 
diff --git a/src/window_gui.h b/src/window_gui.h
index 34eee26362..adb5c10c63 100644
--- a/src/window_gui.h
+++ b/src/window_gui.h
@@ -904,6 +904,7 @@ extern int _scrollbar_size;
 extern byte _scroller_click_timeout;
 
 extern Window *_scrolling_viewport;
+extern Rect _scrolling_viewport_bound;
 extern bool _mouse_hovering;
 
 /** Mouse modes. */