Make capture sources w/o alpha use opaque effect

This fixes an issue primarily with filter rendering: when capturing
windows and displays, their alpha channel is almost always 0, causing
the image to be completely invisible unintentionally.  The original fix
for this for many sources was just to turn off the blending, which would
be fine if you're not rendering any filters, but filters will render to
render targets first, and that lack of alpha will end up carrying over
in to the final image.

This doesn't apply to any mac captures because mac actually seems to set
the alpha channel to 1.

plugins/linux-capture/xcompcap-main.cpp

@@ -501,6 +501,7 @@ void XCompcapMain::tick(float seconds)
 void XCompcapMain::render(gs_effect_t *effect)
 {
 	PLock lock(&p->lock, true);
+	effect = obs_get_opaque_effect();
 
 	if (!lock.isLocked() || !p->tex)
 		return;
@@ -508,13 +509,17 @@ void XCompcapMain::render(gs_effect_t *effect)
 	gs_eparam_t *image = gs_effect_get_param_by_name(effect, "image");
 	gs_effect_set_texture(image, p->tex);
 
-	gs_enable_blending(false);
-	gs_draw_sprite(p->tex, 0, 0, 0);
+	while (gs_effect_loop(effect, "Draw")) {
+		gs_draw_sprite(p->tex, 0, 0, 0);
+	}
 
-	if (p->cursor && p->gltex && p->show_cursor && !p->cursor_outside)
-		xcursor_render(p->cursor);
+	if (p->cursor && p->gltex && p->show_cursor && !p->cursor_outside) {
+		effect = obs_get_default_effect();
 
-	gs_reset_blend_state();
+		while (gs_effect_loop(effect, "Draw")) {
+			xcursor_render(p->cursor);
+		}
+	}
 }
 
 uint32_t XCompcapMain::width()

plugins/linux-capture/xcursor-xcb.c

@@ -96,14 +96,17 @@ void xcb_xcursor_render(xcb_xcursor_t *data)
 	gs_eparam_t *image = gs_effect_get_param_by_name(effect, "image");
 	gs_effect_set_texture(image, data->tex);
 
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_SRCALPHA, GS_BLEND_INVSRCALPHA);
+	gs_enable_color(true, true, true, false);
+
 	gs_matrix_push();
 	gs_matrix_translate3f(data->x_render, data->y_render, 0.0f);
-
-	gs_enable_blending(true);
-	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
 	gs_draw_sprite(data->tex, 0, 0, 0);
-
 	gs_matrix_pop();
+
+	gs_enable_color(true, true, true, true);
+	gs_blend_state_pop();
 }
 
 void xcb_xcursor_offset(xcb_xcursor_t* data, const int x_org, const int y_org)

plugins/linux-capture/xcursor.c

@@ -106,14 +106,17 @@ void xcursor_render(xcursor_t *data) {
 	gs_eparam_t *image = gs_effect_get_param_by_name(effect, "image");
 	gs_effect_set_texture(image, data->tex);
 
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_SRCALPHA, GS_BLEND_INVSRCALPHA);
+	gs_enable_color(true, true, true, false);
+
 	gs_matrix_push();
 	gs_matrix_translate3f(data->render_x, data->render_y, 0.0f);
-
-	gs_enable_blending(True);
-	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
 	gs_draw_sprite(data->tex, 0, 0, 0);
-
 	gs_matrix_pop();
+
+	gs_enable_color(true, true, true, true);
+	gs_blend_state_pop();
 }
 
 void xcursor_offset(xcursor_t* data, int_fast32_t x_org, int_fast32_t y_org)

plugins/linux-capture/xshm-input.c

@@ -431,19 +431,25 @@ static void xshm_video_render(void *vptr, gs_effect_t *effect)
 {
 	XSHM_DATA(vptr);
 
+	effect = obs_get_opaque_effect();
+
 	if (!data->texture)
 		return;
 
 	gs_eparam_t *image = gs_effect_get_param_by_name(effect, "image");
 	gs_effect_set_texture(image, data->texture);
 
-	gs_enable_blending(false);
-	gs_draw_sprite(data->texture, 0, 0, 0);
+	while (gs_effect_loop(effect, "Draw")) {
+		gs_draw_sprite(data->texture, 0, 0, 0);
+	}
 
-	if (data->show_cursor)
-		xcb_xcursor_render(data->cursor);
+	if (data->show_cursor) {
+		effect = obs_get_default_effect();
 
-	gs_reset_blend_state();
+		while (gs_effect_loop(effect, "Draw")) {
+			xcb_xcursor_render(data->cursor);
+		}
+	}
 }
 
 /**

plugins/win-capture/cursor-capture.c

@@ -201,10 +201,17 @@ void cursor_draw(struct cursor_data *data, long x_offset, long y_offset,
 		return;
 
 	if (data->visible && !!data->texture) {
+		gs_blend_state_push();
+		gs_blend_function(GS_BLEND_SRCALPHA, GS_BLEND_INVSRCALPHA);
+		gs_enable_color(true, true, true, false);
+
 		gs_matrix_push();
 		gs_matrix_scale3f(x_scale, y_scale, 1.0f);
 		obs_source_draw(data->texture, x_draw, y_draw, 0, 0, false);
 		gs_matrix_pop();
+
+		gs_enable_color(true, true, true, true);
+		gs_blend_state_pop();
 	}
 }
 

plugins/win-capture/data/opaque.effect

@@ -1,35 +0,0 @@
-uniform float4x4 ViewProj;
-uniform texture2d image;
-
-sampler_state def_sampler {
-	Filter   = Linear;
-	AddressU = Clamp;
-	AddressV = Clamp;
-};
-
-struct VertInOut {
-	float4 pos : POSITION;
-	float2 uv  : TEXCOORD0;
-};
-
-VertInOut VSDefault(VertInOut vert_in)
-{
-	VertInOut vert_out;
-	vert_out.pos = mul(float4(vert_in.pos.xyz, 1.0), ViewProj);
-	vert_out.uv  = vert_in.uv;
-	return vert_out;
-}
-
-float4 PSDraw(VertInOut vert_in) : TARGET
-{
-	return float4(image.Sample(def_sampler, vert_in.uv).rgb, 1.0);
-}
-
-technique Draw
-{
-	pass
-	{
-		vertex_shader = VSDefault(vert_in);
-		pixel_shader  = PSDraw(vert_in);
-	}
-}

plugins/win-capture/duplicator-monitor-capture.c

@@ -194,14 +194,11 @@ static void duplicator_capture_render(void *data, gs_effect_t *effect)
 	if (!texture)
 		return;
 
-	effect = obs_get_default_effect();
+	effect = obs_get_opaque_effect();
 
 	rot = capture->rot;
 
 	while (gs_effect_loop(effect, "Draw")) {
-		gs_enable_blending(false);
-		gs_enable_color(true, true, true, false);
-
 		if (rot != 0) {
 			float x = 0.0f;
 			float y = 0.0f;
@@ -228,12 +225,14 @@ static void duplicator_capture_render(void *data, gs_effect_t *effect)
 
 		if (rot != 0)
 			gs_matrix_pop();
+	}
 
-		gs_enable_blending(true);
-		gs_enable_color(true, true, true, true);
+	if (capture->capture_cursor) {
+		effect = obs_get_default_effect();
 
-		if (capture->capture_cursor)
+		while (gs_effect_loop(effect, "Draw")) {
 			draw_cursor(capture);
+		}
 	}
 }
 

plugins/win-capture/game-capture.c

@@ -1156,23 +1156,22 @@ static void game_capture_render(void *data, gs_effect_t *effect)
 	if (!gc->texture)
 		return;
 
-	effect = obs_get_default_effect();
+	effect = gc->config.allow_transparency ?
+		obs_get_default_effect() : obs_get_opaque_effect();
 
 	while (gs_effect_loop(effect, "Draw")) {
-		if (!gc->config.allow_transparency) {
-			gs_enable_blending(false);
-			gs_enable_color(true, true, true, false);
-		}
-
 		obs_source_draw(gc->texture, 0, 0, 0, 0,
 				gc->global_hook_info->flip);
 
-		if (!gc->config.allow_transparency) {
-			gs_enable_blending(true);
-			gs_enable_color(true, true, true, true);
+		if (gc->config.allow_transparency && gc->config.cursor) {
+			game_capture_render_cursor(gc);
 		}
+	}
 
-		if (gc->config.cursor) {
+	if (!gc->config.allow_transparency && gc->config.cursor) {
+		effect = obs_get_default_effect();
+
+		while (gs_effect_loop(effect, "Draw")) {
 			game_capture_render_cursor(gc);
 		}
 	}

plugins/win-capture/monitor-capture.c

@@ -15,8 +15,6 @@ struct monitor_capture {
 	bool              compatibility;
 
 	struct dc_capture data;
-
-	gs_effect_t       *opaque_effect;
 };
 
 struct monitor_info {
@@ -100,10 +98,7 @@ static void monitor_capture_destroy(void *data)
 	struct monitor_capture *capture = data;
 
 	obs_enter_graphics();
-
 	dc_capture_free(&capture->data);
-	gs_effect_destroy(capture->opaque_effect);
-
 	obs_leave_graphics();
 
 	bfree(capture);
@@ -125,13 +120,8 @@ static void monitor_capture_update(void *data, obs_data_t *settings)
 static void *monitor_capture_create(obs_data_t *settings, obs_source_t *source)
 {
 	struct monitor_capture *capture;
-	gs_effect_t *opaque_effect = create_opaque_effect();
-
-	if (!opaque_effect)
-		return NULL;
 
 	capture = bzalloc(sizeof(struct monitor_capture));
-	capture->opaque_effect = opaque_effect;
 	capture->source = source;
 
 	update_settings(capture, settings);
@@ -156,7 +146,7 @@ static void monitor_capture_tick(void *data, float seconds)
 static void monitor_capture_render(void *data, gs_effect_t *effect)
 {
 	struct monitor_capture *capture = data;
-	dc_capture_render(&capture->data, capture->opaque_effect);
+	dc_capture_render(&capture->data, obs_get_opaque_effect());
 
 	UNUSED_PARAMETER(effect);
 }

plugins/win-capture/window-capture.c

@@ -27,8 +27,6 @@ struct window_capture {
 
 	float                resize_timer;
 
-	gs_effect_t          *opaque_effect;
-
 	HWND                 window;
 	RECT                 last_rect;
 };
@@ -58,14 +56,8 @@ static const char *wc_getname(void)
 
 static void *wc_create(obs_data_t *settings, obs_source_t *source)
 {
-	struct window_capture *wc;
-	gs_effect_t *opaque_effect = create_opaque_effect();
-	if (!opaque_effect)
-		return NULL;
-
-	wc                = bzalloc(sizeof(struct window_capture));
-	wc->source        = source;
-	wc->opaque_effect = opaque_effect;
+	struct window_capture *wc = bzalloc(sizeof(struct window_capture));
+	wc->source = source;
 
 	update_settings(wc, settings);
 	return wc;
@@ -76,16 +68,14 @@ static void wc_destroy(void *data)
 	struct window_capture *wc = data;
 
 	if (wc) {
+		obs_enter_graphics();
 		dc_capture_free(&wc->capture);
+		obs_leave_graphics();
 
 		bfree(wc->title);
 		bfree(wc->class);
 		bfree(wc->executable);
 
-		obs_enter_graphics();
-		gs_effect_destroy(wc->opaque_effect);
-		obs_leave_graphics();
-
 		bfree(wc);
 	}
 }
@@ -198,7 +188,7 @@ static void wc_tick(void *data, float seconds)
 static void wc_render(void *data, gs_effect_t *effect)
 {
 	struct window_capture *wc = data;
-	dc_capture_render(&wc->capture, wc->opaque_effect);
+	dc_capture_render(&wc->capture, obs_get_opaque_effect());
 
 	UNUSED_PARAMETER(effect);
 }