libobs: Fix texture encoder duping frames when queue full

When a texture encoder is working close to its throughput limit and a
stall causes the encoder frame queue to fill up the old implementation
would duplicate frames already in the queue to maintain sync.

This would result in a snowball effect if the encoder wasn't able to
work through the queue fast enough, continually adding more duplicates
keeping the encoder busy while dropping actually new frames that could
be encoded instead.

To fix this we add a flag that allows us to insert "fake" frames into
the queue, those do not go back into the available queue and simply
signal the GPU encoder thread to incremene the PTS without actually
encoding anything. This is required to maintain audio/video sync while
still allowing us to skip frames.

libobs/obs-internal.h

@@ -256,6 +256,7 @@ struct obs_tex_frame {
 	uint64_t lock_key;
 	int count;
 	bool released;
+	bool skip;
 };
 
 struct obs_task_info {

libobs/obs-video-gpu-encode.c

@@ -137,6 +137,18 @@ static void *gpu_encode_thread(void *data)
 			if (skip)
 				continue;
 
+			if (tf.skip) {
+				/* Skip frame but increment PTS to maintain
+				 * sync if encoder already started. */
+				if (encoder->start_ts) {
+					encoder->cur_pts +=
+						encoder->timebase_num *
+						encoder->frame_rate_divisor;
+				}
+
+				continue;
+			}
+
 			if (!encoder->start_ts)
 				encoder->start_ts = timestamp;
 
@@ -182,9 +194,13 @@ static void *gpu_encode_thread(void *data)
 
 		if (--tf.count) {
 			tf.timestamp += interval;
-			deque_push_front(&video->gpu_encoder_queue, &tf,
-					 sizeof(tf));
-
+			deque_push_back(&video->gpu_encoder_queue, &tf,
+					sizeof(tf));
+			if (tf.skip) {
+				video_output_inc_texture_skipped_frames(
+					video->video);
+			}
+		} else if (tf.skip) {
 			video_output_inc_texture_skipped_frames(video->video);
 		} else {
 			deque_push_back(&video->gpu_encoder_avail_queue, &tf,
@@ -243,7 +259,8 @@ bool init_gpu_encoding(struct obs_core_video_mix *video)
 
 		struct obs_tex_frame frame = {.tex = tex,
 					      .tex_uv = tex_uv,
-					      .handle = handle};
+					      .handle = handle,
+					      .skip = false};
 
 		deque_push_back(&video->gpu_encoder_avail_queue, &frame,
 				sizeof(frame));

libobs/obs-video.c

@@ -481,9 +481,8 @@ static inline bool queue_frame(struct obs_core_video_mix *video,
 			       bool raw_active,
 			       struct obs_vframe_info *vframe_info)
 {
-	bool duplicate =
-		!video->gpu_encoder_avail_queue.size ||
-		(video->gpu_encoder_queue.size && vframe_info->count > 1);
+	bool duplicate = video->gpu_encoder_queue.size &&
+			 vframe_info->count > 1;
 
 	if (duplicate) {
 		struct obs_tex_frame *tf =
@@ -500,6 +499,19 @@ static inline bool queue_frame(struct obs_core_video_mix *video,
 		goto finish;
 	}
 
+	/* If no frames are available add a placeholder that will simply
+	 * instruct the gpu encode thread to increment the encoders' pts
+	 * without actually encoding anything. */
+	if (!video->gpu_encoder_avail_queue.size) {
+		struct obs_tex_frame tf = {0};
+		tf.skip = true;
+		tf.count = 1;
+		tf.timestamp = vframe_info->timestamp;
+		deque_push_back(&video->gpu_encoder_queue, &tf, sizeof(tf));
+		os_sem_post(video->gpu_encode_semaphore);
+		goto finish;
+	}
+
 	struct obs_tex_frame tf;
 	deque_pop_front(&video->gpu_encoder_avail_queue, &tf, sizeof(tf));