Merge pull request #9434 from annando/dbclean-limit

Post expiry: Limit for deletions

src/Core/Worker.php

@@ -1394,4 +1394,48 @@ class Worker
 
 		return (bool)$row['jobs'];
 	}
+
+	/**
+	 * Check if the system is inside the defined maintenance window
+	 *
+	 * @return boolean
+	 */
+	public static function isInMaintenanceWindow(bool $check_last_execution = false)
+	{
+		// Calculate the seconds of the start end end of the maintenance window
+		$start = strtotime(DI::config()->get('system', 'maintenance_start')) % 86400;
+		$end = strtotime(DI::config()->get('system', 'maintenance_end')) % 86400;
+
+		Logger::info('Maintenance window', ['start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
+
+		if ($check_last_execution) {
+			// Calculate the window duration
+			$duration = max($start, $end) - min($start, $end);
+
+			// Quit when the last cron execution had been after the previous window
+			$last_cron = DI::config()->get('system', 'last_cron_daily');
+			if ($last_cron + $duration > time()) {
+				Logger::info('The Daily cron had been executed recently', ['last' => date(DateTimeFormat::MYSQL, $last_cron), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
+				return false;
+			}
+		}
+
+		$current = time() % 86400;
+
+		if ($start < $end) {
+			// Execute if we are inside the window
+			$execute = ($current >= $start) && ($current <= $end);
+		} else {
+			// Don't execute if we are outside the window
+			$execute = !(($current > $end) && ($current < $start));
+		}
+
+		if ($execute) {
+			Logger::info('We are inside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
+		} else {
+			Logger::info('We are outside the maintenance window', ['current' => date('H:i:s', $current), 'start' => date('H:i:s', $start), 'end' => date('H:i:s', $end)]);
+		}
+		
+		return $execute;
+	}
 }

src/Worker/Cron.php

@@ -93,8 +93,8 @@ class Cron
 			DI::config()->set('system', 'last_cron_hourly', time());
 		}
 
-		// Daily cron calls
+		// Daily maintenance cron calls
-		if (DI::config()->get('system', 'last_cron_daily', 0) + 86400 < time()) {
+		if (Worker::isInMaintenanceWindow(true)) {
 
 			Worker::add(PRIORITY_LOW, 'UpdateContactBirthdays');
 

src/Worker/ExpirePosts.php

@@ -22,6 +22,7 @@
 namespace Friendica\Worker;
 
 use Friendica\Core\Logger;
+use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Util\DateTimeFormat;
@@ -39,35 +40,58 @@ class ExpirePosts
 			$expire_days_unclaimed = $expire_days;
 		}
 
-		if (!empty($expire_days)) {
+		$limit = DI::config()->get('system', 'dbclean-expire-limit');
-			Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
+		if (empty($limit)) {
-			$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
+			return;
-				(SELECT `uri-id` FROM `thread`
+		}
-				INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts`
-				WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
-					AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin`
-					AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item`
-						WHERE (`item`.`starred` OR (`item`.`resource-id` != '')
-							OR (`item`.`event-id` != '') OR (`item`.`attach` != '')
-							OR `item`.`wall` OR `item`.`origin`
-							OR `uri-id` IN (SELECT `uri-id` FROM `post-category`
-								WHERE `uri-id` = `item`.`uri-id`))
-							AND `item`.`parent` = `thread`.`iid`))", $expire_days);
 
-			Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => DBA::affectedRows()]);
+		if (!empty($expire_days)) {
+			do {
+				Logger::notice('Start deleting expired threads', ['expiry_days' => $expire_days]);
+				$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
+					(SELECT `uri-id` FROM `thread`
+					INNER JOIN `contact` ON `id` = `contact-id` AND NOT `notify_new_posts`
+					WHERE `received` < UTC_TIMESTAMP() - INTERVAL ? DAY
+						AND NOT `mention` AND NOT `starred` AND NOT `wall` AND NOT `origin`
+						AND `thread`.`uid` != 0 AND NOT `iid` IN (SELECT `parent` FROM `item`
+							WHERE (`item`.`starred` OR (`item`.`resource-id` != '')
+								OR (`item`.`event-id` != '') OR (`item`.`attach` != '')
+								OR `item`.`wall` OR `item`.`origin`
+								OR `uri-id` IN (SELECT `uri-id` FROM `post-category`
+									WHERE `uri-id` = `item`.`uri-id`))
+								AND `item`.`parent` = `thread`.`iid`))
+					ORDER BY `id` LIMIT ?", $expire_days, $limit);
+
+				$rows = DBA::affectedRows();
+				Logger::notice('Deleted expired threads', ['result' => $ret, 'rows' => $rows]);
+
+				if (!Worker::isInMaintenanceWindow()) {
+					Logger::notice('We are outside of the maintenance window, quitting');
+					return;
+				}
+			} while ($rows >= $limit);
 		}
 
 		if (!empty($expire_days_unclaimed)) {
 			$expiry_date = DateTimeFormat::utc('now - ' . $expire_days_unclaimed . ' days', DateTimeFormat::MYSQL);
 
-			Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]);
+			do {
-			$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
+				Logger::notice('Start deleting unclaimed public items', ['expiry_days' => $expire_days_unclaimed, 'expired' => $expiry_date]);
-				(SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
+				$ret = DBA::e("DELETE FROM `item-uri` WHERE `id` IN
-					AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?)
+					(SELECT `uri-id` FROM `item` WHERE `gravity` = ? AND `uid` = ? AND `received` < ?
-					AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))",
+						AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` != ?)
-				GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date);
+						AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `item` WHERE `uid` = ? AND `received` > ?))
+					ORDER BY `id` LIMIT ?",
+					GRAVITY_PARENT, 0, $expiry_date, 0, 0, $expiry_date, $limit);
 
-			Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => DBA::affectedRows()]);
+				$rows = DBA::affectedRows();
+				Logger::notice('Deleted unclaimed public items', ['result' => $ret, 'rows' => $rows]);
+
+				if (!Worker::isInMaintenanceWindow()) {
+					Logger::notice('We are outside of the maintenance window, quitting');
+					return;
+				}
+			} while ($rows >= $limit);
 		}
 	}
 }

static/defaults.config.php

@@ -187,7 +187,8 @@ return [
 
 		// dbclean-expire-limit (Integer)
 		// This defines the number of items that are to be deleted in a single call.
-		// Reduce this value when you are getting memory issues.
+		// Reduce this value when you are getting lock issues.
+		// A value of 0 disables the deletion process.
 		'dbclean-expire-limit' => 1000,
 
 		// diaspora_test (Boolean)
@@ -299,6 +300,16 @@ return [
 		// Sets the logging adapter of Friendica globally (monolog, syslog, stream)
 		'logger_config' => 'stream',
 
+		// maintenance_start (String)
+		// Start of the window for the daily maintenance cron call.
+		// The system timezone is used when no timezone is defined here.
+		'maintenance_start' => '01:00 +00:00',
+
+		// maintenance_end (String)
+		// End of the window for the daily maintenance cron call
+		// The system timezone is used when no timezone is defined here.
+		'maintenance_end' => '03:00 +00:00',
+
 		// max_batch_queue (Integer)
 		// Maximum number of batched queue items for a single contact before subsequent messages are discarded.
 		'max_batch_queue' => 1000,