Merge pull request #2836 from jpark37/yuv-image-fix

Fix FFmpeg YUV to RGB conversion by adding extra destination padding to line sizes

deps/media-playback/media-playback/media.c

@@ -216,7 +216,7 @@ static bool mp_media_init_scaling(mp_media_t *m)
 
 	int ret = av_image_alloc(m->scale_pic, m->scale_linesizes,
 				 m->v.decoder->width, m->v.decoder->height,
-				 m->scale_format, 1);
+				 m->scale_format, 32);
 	if (ret < 0) {
 		blog(LOG_WARNING, "MP: Failed to create scale pic data");
 		return false;

libobs/graphics/graphics-ffmpeg.c

@@ -2,6 +2,7 @@
 
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
 #include <libswscale/swscale.h>
 
 #include "../obs-ffmpeg-compat.h"
@@ -122,9 +123,11 @@ static bool ffmpeg_image_reformat_frame(struct ffmpeg_image *info,
 		}
 
 	} else {
+		static const enum AVPixelFormat format = AV_PIX_FMT_BGRA;
+
 		sws_ctx = sws_getContext(info->cx, info->cy, info->format,
-					 info->cx, info->cy, AV_PIX_FMT_BGRA,
-					 SWS_POINT, NULL, NULL, NULL);
+					 info->cx, info->cy, format, SWS_POINT,
+					 NULL, NULL, NULL);
 		if (!sws_ctx) {
 			blog(LOG_WARNING,
 			     "Failed to create scale context "
@@ -133,17 +136,36 @@ static bool ffmpeg_image_reformat_frame(struct ffmpeg_image *info,
 			return false;
 		}
 
+		uint8_t *pointers[4];
+		int linesizes[4];
+		ret = av_image_alloc(pointers, linesizes, info->cx, info->cy,
+				     format, 32);
+		if (ret < 0) {
+			blog(LOG_WARNING, "av_image_alloc failed for '%s': %s",
+			     info->file, av_err2str(ret));
+			sws_freeContext(sws_ctx);
+			return false;
+		}
+
 		ret = sws_scale(sws_ctx, (const uint8_t *const *)frame->data,
-				frame->linesize, 0, info->cy, &out, &linesize);
+				frame->linesize, 0, info->cy, pointers,
+				linesizes);
 		sws_freeContext(sws_ctx);
 
 		if (ret < 0) {
 			blog(LOG_WARNING, "sws_scale failed for '%s': %s",
 			     info->file, av_err2str(ret));
+			av_freep(pointers);
 			return false;
 		}
 
-		info->format = AV_PIX_FMT_BGRA;
+		for (size_t y = 0; y < (size_t)info->cy; y++)
+			memcpy(out + y * linesize,
+			       pointers[0] + y * linesizes[0], linesize);
+
+		av_freep(pointers);
+
+		info->format = format;
 	}
 
 	return true;

libobs/media-io/video-scaler-ffmpeg.c

@@ -18,11 +18,15 @@
 #include "../util/bmem.h"
 #include "video-scaler.h"
 
+#include <libavutil/imgutils.h>
 #include <libswscale/swscale.h>
 
 struct video_scaler {
 	struct SwsContext *swscale;
 	int src_height;
+	int dst_height;
+	uint8_t *dst_pointers[4];
+	int dst_linesizes[4];
 };
 
 static inline enum AVPixelFormat
@@ -138,6 +142,15 @@ int video_scaler_create(video_scaler_t **scaler_out,
 
 	scaler = bzalloc(sizeof(struct video_scaler));
 	scaler->src_height = src->height;
+	scaler->dst_height = dst->height;
+
+	ret = av_image_alloc(scaler->dst_pointers, scaler->dst_linesizes,
+			     dst->width, dst->height, format_dst, 32);
+	if (ret < 0) {
+		blog(LOG_WARNING,
+		     "video_scaler_create: av_image_alloc failed: %d", ret);
+		goto fail;
+	}
 
 	scaler->swscale = sws_getCachedContext(NULL, src->width, src->height,
 					       format_src, dst->width,
@@ -169,6 +182,10 @@ void video_scaler_destroy(video_scaler_t *scaler)
 {
 	if (scaler) {
 		sws_freeContext(scaler->swscale);
+
+		if (scaler->dst_pointers[0])
+			av_freep(scaler->dst_pointers);
+
 		bfree(scaler);
 	}
 }
@@ -182,13 +199,37 @@ bool video_scaler_scale(video_scaler_t *scaler, uint8_t *output[],
 		return false;
 
 	int ret = sws_scale(scaler->swscale, input, (const int *)in_linesize, 0,
-			    scaler->src_height, output,
-			    (const int *)out_linesize);
+			    scaler->src_height, scaler->dst_pointers,
+			    scaler->dst_linesizes);
 	if (ret <= 0) {
 		blog(LOG_ERROR, "video_scaler_scale: sws_scale failed: %d",
 		     ret);
 		return false;
 	}
 
+	const size_t height = scaler->dst_height;
+	for (size_t plane = 0; plane < 4; ++plane) {
+		if (!scaler->dst_pointers[plane])
+			continue;
+
+		const size_t scaled_linesize = scaler->dst_linesizes[plane];
+		const size_t plane_linesize = out_linesize[plane];
+		uint8_t *dst = output[plane];
+		const uint8_t *src = scaler->dst_pointers[plane];
+		if (scaled_linesize == plane_linesize) {
+			memcpy(dst, src, scaled_linesize * height);
+		} else {
+			size_t linesize = scaled_linesize;
+			if (linesize > plane_linesize)
+				linesize = plane_linesize;
+
+			for (size_t y = 0; y < height; y++) {
+				memcpy(dst, src, linesize);
+				dst += plane_linesize;
+				src += scaled_linesize;
+			}
+		}
+	}
+
 	return true;
 }