(API Change) libobs: Add bicubic/lanczos scaling

This adds bicubic and lanczos scaling capability to libobs to improve
scaling quality and sharpness when the output resolution has to be
scaled relative to the base resolution.  Bilinear is also available,
although bilinear has rather poor quality and causes scaling to appear
blurry.

If the output resolution is close to the base resolution, then bilinear
is used instead as an optimization, as there's no need to use these
shaders if scaling is not in use.

The Bicubic and Lanczos effects are also exposed via exported function
to allow the ability to use those shaders in plugin modules if desired.

The API change adds a variable 'scale_type' to the obs_video_info
structure that allows the user interface to choose what type of scaling
filter should be used.

libobs/data/bicubic_scale.effect

@@ -0,0 +1,139 @@
+/*
+ * bicubic sharper (better for downscaling)
+ * note - this shader is adapted from the GPL bsnes shader, very good stuff
+ * there.
+ */
+
+uniform float4x4 ViewProj;
+uniform texture2d image;
+uniform float4x4 color_matrix;
+uniform float3 color_range_min = {0.0, 0.0, 0.0};
+uniform float3 color_range_max = {1.0, 1.0, 1.0};
+uniform float2 base_dimension_i;
+
+sampler_state textureSampler {
+	Filter    = Linear;
+	AddressU  = Clamp;
+	AddressV  = Clamp;
+};
+
+struct VertData {
+	float4 pos : POSITION;
+	float2 uv  : TEXCOORD0;
+};
+
+VertData VSDefault(VertData input)
+{
+	VertData vert_out;
+	vert_out.pos = mul(float4(input.pos.xyz, 1.0), ViewProj);
+	vert_out.uv  = input.uv;
+	return vert_out;
+}
+
+float weight(float x)
+{
+	float ax = abs(x);
+
+	/* Sharper version.  May look better in some cases. */
+	const float B = 0.0;
+	const float C = 0.75;
+
+	if (ax < 1.0)
+		return (pow(x, 2.0) *
+			((12.0 - 9.0 * B - 6.0 * C) * ax +
+				(-18.0 + 12.0 * B + 6.0 * C)) +
+				(6.0 - 2.0 * B))
+			/ 6.0;
+	else if ((ax >= 1.0) && (ax < 2.0))
+		return (pow(x, 2.0) *
+			((-B - 6.0 * C) * ax + (6.0 * B + 30.0 * C)) +
+				(-12.0 * B - 48.0 * C) * ax +
+				(8.0 * B + 24.0 * C))
+			/ 6.0;
+	else
+		return 0.0;
+}
+
+float4 weight4(float x)
+{
+	return float4(
+		weight(x - 2.0),
+		weight(x - 1.0),
+		weight(x),
+		weight(x + 1.0));
+}
+
+float4 pixel(float xpos, float ypos)
+{
+	return image.Sample(textureSampler, float2(xpos, ypos));
+}
+
+float4 get_line(float ypos, float4 xpos, float4 linetaps)
+{
+	return
+		pixel(xpos.r, ypos) * linetaps.r +
+		pixel(xpos.g, ypos) * linetaps.g +
+		pixel(xpos.b, ypos) * linetaps.b +
+		pixel(xpos.a, ypos) * linetaps.a;
+}
+
+float4 DrawBicubic(VertData input)
+{
+	float2 stepxy = base_dimension_i;
+	float2 pos = input.uv + stepxy * 0.5;
+	float2 f = frac(pos / stepxy);
+
+	float4 rowtaps = weight4(1.0 - f.x);
+	float4 coltaps = weight4(1.0 - f.y);
+
+	/* make sure all taps added together is exactly 1.0, otherwise some
+ 	 * (very small) distortion can occur */
+	rowtaps /= rowtaps.r + rowtaps.g + rowtaps.b + rowtaps.a;
+	coltaps /= coltaps.r + coltaps.g + coltaps.b + coltaps.a;
+
+	float2 xystart = (-1.5 - f) * stepxy + pos;
+	float4 xpos = float4(
+		xystart.x,
+		xystart.x + stepxy.x,
+		xystart.x + stepxy.x * 2.0,
+		xystart.x + stepxy.x * 3.0
+	);
+
+	return
+		get_line(xystart.y                 , xpos, rowtaps) * coltaps.r +
+		get_line(xystart.y + stepxy.y      , xpos, rowtaps) * coltaps.g +
+		get_line(xystart.y + stepxy.y * 2.0, xpos, rowtaps) * coltaps.b +
+		get_line(xystart.y + stepxy.y * 3.0, xpos, rowtaps) * coltaps.a;
+}
+
+float4 PSDrawBicubicRGBA(VertData input) : TARGET
+{
+	return DrawBicubic(input);
+}
+
+float4 PSDrawBicubicMatrix(VertData input) : TARGET
+{
+	float4 rgba = DrawBicubic(input);
+	float4 yuv;
+
+	yuv.xyz = clamp(rgba.xyz, color_range_min, color_range_max);
+	return saturate(mul(float4(yuv.xyz, 1.0), color_matrix));
+}
+
+technique Draw
+{
+	pass
+	{
+		vertex_shader = VSDefault(input);
+		pixel_shader  = PSDrawBicubicRGBA(input);
+	}
+}
+
+technique DrawMatrix
+{
+	pass
+	{
+		vertex_shader = VSDefault(input);
+		pixel_shader  = PSDrawBicubicMatrix(input);
+	}
+}

libobs/data/lanczos_scale.effect

@@ -0,0 +1,139 @@
+/*
+ * lanczos sharper
+ * note - this shader is adapted from the GPL bsnes shader, very good stuff
+ * there.
+ */
+
+uniform float4x4 ViewProj;
+uniform texture2d image;
+uniform float4x4 color_matrix;
+uniform float3 color_range_min = {0.0, 0.0, 0.0};
+uniform float3 color_range_max = {1.0, 1.0, 1.0};
+uniform float2 base_dimension_i;
+
+sampler_state textureSampler
+{
+	AddressU  = Clamp;
+	AddressV  = Clamp;
+	Filter    = Linear;
+};
+
+struct VertData {
+	float4 pos : POSITION;
+	float2 uv  : TEXCOORD0;
+};
+
+VertData VSDefault(VertData input)
+{
+	VertData vert_out;
+	vert_out.pos = mul(float4(input.pos.xyz, 1.0), ViewProj);
+	vert_out.uv  = input.uv;
+	return vert_out;
+}
+
+float sinc(float x)
+{
+	const float PIval = 3.1415926535897932384626433832795;
+	return sin(x * PIval) / (x * PIval);
+}
+
+float weight(float x, float radius)
+{
+	float ax = abs(x);
+	if (x == 0.0)
+		return 1.0;
+	else if (ax < radius)
+		return sinc(x) * sinc(x / radius);
+	else
+		return 0.0;
+}
+
+float3 weight3(float x)
+{
+	return float3(
+		weight(x * 2.0 + 0.0 * 2.0 - 3.0, 3.0),
+		weight(x * 2.0 + 1.0 * 2.0 - 3.0, 3.0),
+		weight(x * 2.0 + 2.0 * 2.0 - 3.0, 3.0));
+}
+
+float4 pixel(float xpos, float ypos)
+{
+	return image.Sample(textureSampler, float2(xpos, ypos));
+}
+
+float4 get_line(float ypos, float3 xpos1, float3 xpos2, float3 rowtap1,
+		float3 rowtap2)
+{
+	return
+		pixel(xpos1.r, ypos) * rowtap1.r +
+		pixel(xpos1.g, ypos) * rowtap2.r +
+		pixel(xpos1.b, ypos) * rowtap1.g +
+		pixel(xpos2.r, ypos) * rowtap2.g +
+		pixel(xpos2.g, ypos) * rowtap1.b +
+		pixel(xpos2.b, ypos) * rowtap2.b;
+}
+
+float4 DrawLanczos(VertData input)
+{
+	float2 stepxy = base_dimension_i;
+	float2 pos = input.uv + stepxy * 0.5;
+	float2 f = frac(pos / stepxy);
+
+	float3 rowtap1 = weight3((1.0 - f.x) / 2.0);
+	float3 rowtap2 = weight3((1.0 - f.x) / 2.0 + 0.5);
+	float3 coltap1 = weight3((1.0 - f.y) / 2.0);
+	float3 coltap2 = weight3((1.0 - f.y) / 2.0 + 0.5);
+
+	/* make sure all taps added together is exactly 1.0, otherwise some
+         * (very small) distortion can occur */
+	float suml = rowtap1.r + rowtap1.g + rowtap1.b + rowtap2.r + rowtap2.g + rowtap2.b;
+	float sumc = coltap1.r + coltap1.g + coltap1.b + coltap2.r + coltap2.g + coltap2.b;
+	rowtap1 /= suml;
+	rowtap2 /= suml;
+	coltap1 /= sumc;
+	coltap2 /= sumc;
+
+	float2 xystart = (-2.5 - f) * stepxy + pos;
+	float3 xpos1 = float3(xystart.x                 , xystart.x + stepxy.x      , xystart.x + stepxy.x * 2.0);
+	float3 xpos2 = float3(xystart.x + stepxy.x * 3.0, xystart.x + stepxy.x * 4.0, xystart.x + stepxy.x * 5.0);
+
+	return
+		get_line(xystart.y                 , xpos1, xpos2, rowtap1, rowtap2) * coltap1.r +
+		get_line(xystart.y + stepxy.y      , xpos1, xpos2, rowtap1, rowtap2) * coltap2.r +
+		get_line(xystart.y + stepxy.y * 2.0, xpos1, xpos2, rowtap1, rowtap2) * coltap1.g +
+		get_line(xystart.y + stepxy.y * 3.0, xpos1, xpos2, rowtap1, rowtap2) * coltap2.g +
+		get_line(xystart.y + stepxy.y * 4.0, xpos1, xpos2, rowtap1, rowtap2) * coltap1.b +
+		get_line(xystart.y + stepxy.y * 5.0, xpos1, xpos2, rowtap1, rowtap2) * coltap2.b;
+}
+
+float4 PSDrawLanczosRGBA(VertData input) : TARGET
+{
+	return DrawLanczos(input);
+}
+
+float4 PSDrawLanczosMatrix(VertData input) : TARGET
+{
+	float4 rgba = DrawLanczos(input);
+	float4 yuv;
+
+	yuv.xyz = clamp(rgba.xyz, color_range_min, color_range_max);
+	return saturate(mul(float4(yuv.xyz, 1.0), color_matrix));
+}
+
+technique Draw
+{
+	pass
+	{
+		vertex_shader = VSDefault(input);
+		pixel_shader  = PSDrawLanczosRGBA(input);
+	}
+}
+
+technique DrawMatrix
+{
+	pass
+	{
+		vertex_shader = VSDefault(input);
+		pixel_shader  = PSDrawLanczosMatrix(input);
+	}
+}

libobs/obs-internal.h

@@ -145,6 +145,8 @@ struct obs_core_video {
 	gs_effect_t                     *default_rect_effect;
 	gs_effect_t                     *solid_effect;
 	gs_effect_t                     *conversion_effect;
+	gs_effect_t                     *bicubic_effect;
+	gs_effect_t                     *lanczos_effect;
 	gs_stagesurf_t                  *mapped_surface;
 	int                             cur_texture;
 
@@ -164,6 +166,7 @@ struct obs_core_video {
 	uint32_t                        base_width;
 	uint32_t                        base_height;
 	float                           color_matrix[16];
+	enum obs_scale_type             scale_type;
 
 	struct obs_display              main_display;
 };

libobs/obs-video.c

@@ -109,6 +109,44 @@ static inline void render_main_texture(struct obs_core_video *video,
 	video->textures_rendered[cur_texture] = true;
 }
 
+static inline gs_effect_t *get_scale_effect_internal(
+		struct obs_core_video *video)
+{
+	switch (video->scale_type) {
+	case OBS_SCALE_BILINEAR: return video->default_effect;
+	case OBS_SCALE_LANCZOS:  return video->lanczos_effect;
+	case OBS_SCALE_BICUBIC:;
+	}
+
+	return video->bicubic_effect;
+}
+
+static inline bool resolution_close(struct obs_core_video *video,
+		uint32_t width, uint32_t height)
+{
+	long width_cmp  = (long)video->base_width  - (long)width;
+	long height_cmp = (long)video->base_height - (long)height;
+
+	return labs(width_cmp) <= 16 && labs(height_cmp) <= 16;
+}
+
+static inline gs_effect_t *get_scale_effect(struct obs_core_video *video,
+		uint32_t width, uint32_t height)
+{
+	if (resolution_close(video, width, height)) {
+		return video->default_effect;
+	} else {
+		/* if the scale method couldn't be loaded, use either bicubic
+		 * or bilinear by default */
+		gs_effect_t *effect = get_scale_effect_internal(video);
+		if (!effect)
+			effect = !!video->bicubic_effect ?
+				video->bicubic_effect :
+				video->default_effect;
+		return effect;
+	}
+}
+
 static inline void render_output_texture(struct obs_core_video *video,
 		int cur_texture, int prev_texture)
 {
@@ -116,13 +154,19 @@ static inline void render_output_texture(struct obs_core_video *video,
 	gs_texture_t *target  = video->output_textures[cur_texture];
 	uint32_t     width   = gs_texture_get_width(target);
 	uint32_t     height  = gs_texture_get_height(target);
+	struct vec2  base_i;
 
-	/* TODO: replace with actual downscalers or unpackers */
-	gs_effect_t    *effect  = video->default_effect;
+	vec2_set(&base_i,
+		1.0f / (float)video->base_width,
+		1.0f / (float)video->base_height);
+
+	gs_effect_t    *effect  = get_scale_effect(video, width, height);
 	gs_technique_t *tech    = gs_effect_get_technique(effect, "DrawMatrix");
 	gs_eparam_t    *image   = gs_effect_get_param_by_name(effect, "image");
 	gs_eparam_t    *matrix  = gs_effect_get_param_by_name(effect,
 			"color_matrix");
+	gs_eparam_t    *bres_i  = gs_effect_get_param_by_name(effect,
+			"base_dimension_i");
 	size_t      passes, i;
 
 	if (!video->textures_rendered[prev_texture])
@@ -131,6 +175,9 @@ static inline void render_output_texture(struct obs_core_video *video,
 	gs_set_render_target(target, NULL);
 	set_render_size(width, height);
 
+	if (bres_i)
+		gs_effect_set_vec2(bres_i, &base_i);
+
 	gs_effect_set_val(matrix, video->color_matrix, sizeof(float) * 16);
 	gs_effect_set_texture(image, texture);
 

libobs/obs.c

@@ -244,6 +244,16 @@ static int obs_init_graphics(struct obs_video_info *ovi)
 			NULL);
 	bfree(filename);
 
+	filename = find_libobs_data_file("bicubic_scale.effect");
+	video->bicubic_effect = gs_effect_create_from_file(filename,
+			NULL);
+	bfree(filename);
+
+	filename = find_libobs_data_file("lanczos_scale.effect");
+	video->lanczos_effect = gs_effect_create_from_file(filename,
+			NULL);
+	bfree(filename);
+
 	if (!video->default_effect)
 		success = false;
 	if (gs_get_device_type() == GS_DEVICE_OPENGL) {
@@ -293,6 +303,7 @@ static int obs_init_video(struct obs_video_info *ovi)
 	video->output_width   = ovi->output_width;
 	video->output_height  = ovi->output_height;
 	video->gpu_conversion = ovi->gpu_conversion;
+	video->scale_type     = ovi->scale_type;
 
 	set_video_matrix(video, ovi);
 
@@ -399,6 +410,8 @@ static void obs_free_graphics(void)
 		gs_effect_destroy(video->default_rect_effect);
 		gs_effect_destroy(video->solid_effect);
 		gs_effect_destroy(video->conversion_effect);
+		gs_effect_destroy(video->bicubic_effect);
+		gs_effect_destroy(video->lanczos_effect);
 		video->default_effect = NULL;
 
 		gs_leave_context();
@@ -756,6 +769,7 @@ bool obs_get_video_info(struct obs_video_info *ovi)
 	ovi->base_width    = video->base_width;
 	ovi->base_height   = video->base_height;
 	ovi->gpu_conversion= video->gpu_conversion;
+	ovi->scale_type    = video->scale_type;
 	ovi->colorspace    = info->colorspace;
 	ovi->range         = info->range;
 	ovi->output_width  = info->width;

libobs/obs.h

@@ -104,6 +104,12 @@ enum obs_allow_direct_render {
 	OBS_ALLOW_DIRECT_RENDERING,
 };
 
+enum obs_scale_type {
+	OBS_SCALE_BICUBIC,
+	OBS_SCALE_BILINEAR,
+	OBS_SCALE_LANCZOS
+};
+
 /**
  * Used with scene items to indicate the type of bounds to use for scene items.
  * Mostly determines how the image will be scaled within those bounds, or
@@ -162,6 +168,8 @@ struct obs_video_info {
 
 	enum video_colorspace colorspace;  /**< YUV type (if YUV) */
 	enum video_range_type range;       /**< YUV range (if YUV) */
+
+	enum obs_scale_type scale_type;    /**< How to scale if scaling */
 };
 
 /**
@@ -480,6 +488,12 @@ EXPORT gs_effect_t *obs_get_default_rect_effect(void);
 /** Returns the solid effect for drawing solid colors */
 EXPORT gs_effect_t *obs_get_solid_effect(void);
 
+/** Returns the bicubic scaling effect */
+EXPORT gs_effect_t *obs_get_bicubic_effect(void);
+
+/** Returns the lanczos scaling effect */
+EXPORT gs_effect_t *obs_get_lanczos_effect(void);
+
 /** Returns the primary obs signal handler */
 EXPORT signal_handler_t *obs_get_signal_handler(void);