(API Change) Add colorspace info to obs_video_info

This was an important change because we were originally using an
hard-coded 709/partial range color matrix for the output, which was
causing problems for people wanting to use different formats or color
spaces.  This will now automatically generate the color matrix depending
on the format, color space, and range, or use an identity matrix if the
video format is RGB instead of YUV.

libobs/obs-internal.h

@@ -163,6 +163,7 @@ struct obs_core_video {
 	uint32_t                        output_height;
 	uint32_t                        base_width;
 	uint32_t                        base_height;
+	float                           color_matrix[16];
 
 	struct obs_display              main_display;
 };

libobs/obs-video.c

@@ -131,16 +131,7 @@ static inline void render_output_texture(struct obs_core_video *video,
 	gs_set_render_target(target, NULL);
 	set_render_size(width, height);
 
-	/* TODO: replace with programmable code */
-	const float mat_val[16] =
-	{
-		-0.100644f, -0.338572f,  0.439216f,  0.501961f,
-		 0.182586f,  0.614231f,  0.062007f,  0.062745f,
-		 0.439216f, -0.398942f, -0.040274f,  0.501961f,
-		 0.000000f,  0.000000f,  0.000000f,  1.000000f
-	};
-
-	gs_effect_set_val(matrix, mat_val, sizeof(mat_val));
+	gs_effect_set_val(matrix, video->color_matrix, sizeof(float) * 16);
 	gs_effect_set_texture(image, texture);
 
 	gs_enable_blending(false);

libobs/obs.c

@@ -17,6 +17,7 @@
 
 #include <inttypes.h>
 
+#include "graphics/matrix4.h"
 #include "callback/calldata.h"
 
 #include "obs.h"
@@ -48,6 +49,8 @@ static inline void make_video_info(struct video_output_info *vi,
 	vi->fps_den = ovi->fps_den;
 	vi->width   = ovi->output_width;
 	vi->height  = ovi->output_height;
+	vi->range   = ovi->range;
+	vi->colorspace = ovi->colorspace;
 }
 
 #define PIXEL_SIZE 4
@@ -256,6 +259,28 @@ static int obs_init_graphics(struct obs_video_info *ovi)
 	return success ? OBS_VIDEO_SUCCESS : OBS_VIDEO_FAIL;
 }
 
+static inline void set_video_matrix(struct obs_core_video *video,
+		struct obs_video_info *ovi)
+{
+	struct matrix4 mat;
+	struct vec4 r_row;
+
+	if (format_is_yuv(ovi->output_format)) {
+		video_format_get_parameters(ovi->colorspace, ovi->range,
+				(float*)&mat, NULL, NULL);
+		matrix4_inv(&mat, &mat);
+
+		/* swap R and G */
+		r_row = mat.x;
+		mat.x = mat.y;
+		mat.y = r_row;
+	} else {
+		matrix4_identity(&mat);
+	}
+
+	memcpy(video->color_matrix, &mat, sizeof(float) * 16);
+}
+
 static int obs_init_video(struct obs_video_info *ovi)
 {
 	struct obs_core_video *video = &obs->video;
@@ -269,6 +294,8 @@ static int obs_init_video(struct obs_video_info *ovi)
 	video->output_height  = ovi->output_height;
 	video->gpu_conversion = ovi->gpu_conversion;
 
+	set_video_matrix(video, ovi);
+
 	errorcode = video_output_open(&video->video, &vi);
 
 	if (errorcode != VIDEO_OUTPUT_SUCCESS) {
@@ -728,6 +755,9 @@ bool obs_get_video_info(struct obs_video_info *ovi)
 	memset(ovi, 0, sizeof(struct obs_video_info));
 	ovi->base_width    = video->base_width;
 	ovi->base_height   = video->base_height;
+	ovi->gpu_conversion= video->gpu_conversion;
+	ovi->colorspace    = info->colorspace;
+	ovi->range         = info->range;
 	ovi->output_width  = info->width;
 	ovi->output_height = info->height;
 	ovi->output_format = info->format;

libobs/obs.h

@@ -159,6 +159,9 @@ struct obs_video_info {
 
 	/** Use shaders to convert to different color formats */
 	bool                gpu_conversion;
+
+	enum video_colorspace colorspace;  /**< YUV type (if YUV) */
+	enum video_range_type range;       /**< YUV range (if YUV) */
 };
 
 /**

obs/window-basic-main.cpp

@@ -1352,6 +1352,8 @@ int OBSBasic::ResetVideo()
 	ovi.output_height  = (uint32_t)config_get_uint(basicConfig,
 			"Video", "OutputCY");
 	ovi.output_format  = VIDEO_FORMAT_NV12;
+	ovi.colorspace     = VIDEO_CS_709;
+	ovi.range          = VIDEO_RANGE_FULL;
 	ovi.adapter        = 0;
 	ovi.gpu_conversion = true;