From 5fe6feb59a83b8523f7aedf8f1c9f6b43403c3e0 Mon Sep 17 00:00:00 2001
From: Philip Haynes <singularity.haynes@gmail.com>
Date: Thu, 13 Sep 2018 07:52:19 -0500
Subject: [PATCH] linux-capture: Improve XComposite capture robustness

Adds support for windows with alpha channels as well as swapping red
and blue when using GS_BGRX. glXFBConfig now attempts to inherit
its format from the captured window when possible.

Change log message
---
 plugins/linux-capture/xcompcap-main.cpp | 75 ++++++++++++++++++++++---
 1 file changed, 66 insertions(+), 9 deletions(-)

diff --git a/plugins/linux-capture/xcompcap-main.cpp b/plugins/linux-capture/xcompcap-main.cpp
index ba7986f09..c01a604a7 100644
--- a/plugins/linux-capture/xcompcap-main.cpp
+++ b/plugins/linux-capture/xcompcap-main.cpp
@@ -148,6 +148,7 @@ struct XCompcapMain_private
 	bool lockX;
 	bool include_border;
 	bool exclude_alpha;
+	bool draw_opaque;
 
 	double window_check_time = 0.0;
 
@@ -301,6 +302,7 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 		p->show_cursor = obs_data_get_bool(settings, "show_cursor");
 		p->include_border = obs_data_get_bool(settings, "include_border");
 		p->exclude_alpha = obs_data_get_bool(settings, "exclude_alpha");
+		p->draw_opaque = false;
 	} else {
 		p->win = prevWin;
 	}
@@ -347,6 +349,18 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 		cf = GS_BGRX;
 	}
 
+	bool has_alpha = true;
+
+	if (attr.depth < 32) {
+		cf = GS_BGRX;
+		has_alpha = false;
+	}
+
+	if (cf == GS_BGRX) {
+		p->swapRedBlue = !p->swapRedBlue;
+		p->draw_opaque = true;
+	}
+
 	p->border = attr.border_width;
 
 	if (p->include_border) {
@@ -395,16 +409,25 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 		glBindTexture(GL_TEXTURE_2D, 0);
 	}
 
-	const int attrs[] =
+	const int attrs_alpha[] =
 	{
 		GLX_BIND_TO_TEXTURE_RGBA_EXT, GL_TRUE,
 		GLX_DRAWABLE_TYPE, GLX_PIXMAP_BIT,
 		GLX_BIND_TO_TEXTURE_TARGETS_EXT, GLX_TEXTURE_2D_BIT_EXT,
 		GLX_ALPHA_SIZE, 8,
-		GLX_DOUBLEBUFFER, GL_FALSE,
 		None
 	};
 
+	const int attrs_no_alpha[] =
+	{
+		GLX_BIND_TO_TEXTURE_RGB_EXT, GL_TRUE,
+		GLX_DRAWABLE_TYPE, GLX_PIXMAP_BIT,
+		GLX_BIND_TO_TEXTURE_TARGETS_EXT, GLX_TEXTURE_2D_BIT_EXT,
+		None
+	};
+
+	const int *attrs = has_alpha ? attrs_alpha : attrs_no_alpha;
+
 	int nelem = 0;
 	GLXFBConfig* configs = glXChooseFBConfig(xdisp,
 			XCompcap::getRootWindowScreen(attr.root),
@@ -418,8 +441,26 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 		return;
 	}
 
-	glXGetFBConfigAttrib(xdisp, configs[0], GLX_Y_INVERTED_EXT, &nelem);
-	p->inverted = nelem != 0;
+	GLXFBConfig config;
+	bool found = false;
+	for (int i = 0; i < nelem; i++) {
+		int visual;
+		config = configs[i];
+		glXGetFBConfigAttrib(xdisp, config, GLX_VISUAL_ID, &visual);
+		if ((int)attr.visual->visualid == visual) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		config = configs[0];
+		p->draw_opaque = true;
+	}
+
+	int inverted;
+	glXGetFBConfigAttrib(xdisp, config, GLX_Y_INVERTED_EXT, &inverted);
+	p->inverted = inverted != 0;
 
 	xlock.resetError();
 
@@ -433,14 +474,23 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 		return;
 	}
 
-	const int attribs[] =
+	const int attribs_alpha[] =
 	{
 		GLX_TEXTURE_TARGET_EXT, GLX_TEXTURE_2D_EXT,
 		GLX_TEXTURE_FORMAT_EXT, GLX_TEXTURE_FORMAT_RGBA_EXT,
 		None
 	};
 
-	p->glxpixmap = glXCreatePixmap(xdisp, configs[0], p->pixmap, attribs);
+	const int attribs_no_alpha[] =
+	{
+		GLX_TEXTURE_TARGET_EXT, GLX_TEXTURE_2D_EXT,
+		GLX_TEXTURE_FORMAT_EXT, GLX_TEXTURE_FORMAT_RGB_EXT,
+		None
+	};
+
+	const int *attribs = has_alpha ? attribs_alpha : attribs_no_alpha;
+
+	p->glxpixmap = glXCreatePixmap(xdisp, config, p->pixmap, attribs);
 
 	if (xlock.gotError()) {
 		blog(LOG_ERROR, "glXCreatePixmap failed: %s",
@@ -466,10 +516,14 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 	if (!p->windowName.empty()) {
 		blog(LOG_INFO, "[window-capture: '%s'] update settings:\n"
 				"\ttitle: %s\n"
-				"\tclass: %s",
+				"\tclass: %s\n"
+				"\tHas alpha: %s\n"
+				"\tFound exact GLXFBConfig: %s",
 				obs_source_get_name(p->source),
 				XCompcap::getWindowName(p->win).c_str(),
-				XCompcap::getWindowClass(p->win).c_str());
+				XCompcap::getWindowClass(p->win).c_str(),
+				has_alpha ? "yes" : "no",
+				found ? "yes" : "no");
 		blog(LOG_DEBUG, "\n"
 				"\tid:    %s",
 				std::to_string((long long)p->win).c_str());
@@ -563,7 +617,10 @@ void XCompcapMain::render(gs_effect_t *effect)
 
 	PLock lock(&p->lock, true);
 
-	effect = obs_get_base_effect(OBS_EFFECT_OPAQUE);
+	if (p->draw_opaque)
+		effect = obs_get_base_effect(OBS_EFFECT_OPAQUE);
+	else
+		effect = obs_get_base_effect(OBS_EFFECT_DEFAULT);
 
 	if (!lock.isLocked() || !p->tex)
 		return;