From 65daf2c86dc78fb42a7c9267275494971cab47e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ste=CC=81phane=20Cottin?= <stephane.cottin@vixns.com>
Date: Mon, 2 Nov 2020 09:14:04 +0100
Subject: [PATCH] decklink: Fix format detection loop

Ignore color space change in format detection to fix endless loop.
Fix #3511
Fix #3277
---
 plugins/decklink/decklink-device-instance.cpp | 36 +++++++++----------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/plugins/decklink/decklink-device-instance.cpp b/plugins/decklink/decklink-device-instance.cpp
index 170b31f96..a49dd4e6f 100644
--- a/plugins/decklink/decklink-device-instance.cpp
+++ b/plugins/decklink/decklink-device-instance.cpp
@@ -618,13 +618,6 @@ HRESULT STDMETHODCALLTYPE DeckLinkDeviceInstance::VideoInputFormatChanged(
 	BMDVideoInputFormatChangedEvents events, IDeckLinkDisplayMode *newMode,
 	BMDDetectedVideoInputFormatFlags detectedSignalFlags)
 {
-	input->PauseStreams();
-
-	mode->SetMode(newMode);
-
-	if (events & bmdVideoInputDisplayModeChanged) {
-		displayMode = mode->GetDisplayMode();
-	}
 
 	if (events & bmdVideoInputColorspaceChanged) {
 		switch (detectedSignalFlags) {
@@ -639,21 +632,26 @@ HRESULT STDMETHODCALLTYPE DeckLinkDeviceInstance::VideoInputFormatChanged(
 		}
 	}
 
-	const HRESULT videoResult = input->EnableVideoInput(
-		displayMode, pixelFormat, bmdVideoInputEnableFormatDetection);
-	if (videoResult != S_OK) {
-		LOG(LOG_ERROR, "Failed to enable video input");
-		input->StopStreams();
-		FinalizeStream();
+	if (events & bmdVideoInputDisplayModeChanged) {
+		input->PauseStreams();
+		mode->SetMode(newMode);
+		displayMode = mode->GetDisplayMode();
 
-		return E_FAIL;
+		const HRESULT videoResult = input->EnableVideoInput(
+			displayMode, pixelFormat,
+			bmdVideoInputEnableFormatDetection);
+		if (videoResult != S_OK) {
+			LOG(LOG_ERROR, "Failed to enable video input");
+			input->StopStreams();
+			FinalizeStream();
+
+			return E_FAIL;
+		}
+		SetupVideoFormat(mode);
+		input->FlushStreams();
+		input->StartStreams();
 	}
 
-	SetupVideoFormat(mode);
-
-	input->FlushStreams();
-	input->StartStreams();
-
 	return S_OK;
 }