From 385a0c8e8cc7e746794616a59710bf83f1ad349c Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Jun 2024 03:20:22 +0000 Subject: [PATCH] Lock for "optimize" / expire in chunks --- database.sql | 54 ++++++------ src/DI.php | 2 +- src/Database/DBA.php | 22 +++++ src/Database/Database.php | 49 ++++++++++- src/Worker/ExpirePosts.php | 172 ++++++++++++++++++++++--------------- 5 files changed, 202 insertions(+), 97 deletions(-) diff --git a/database.sql b/database.sql index 6c16deb30d..979320cc7c 100644 --- a/database.sql +++ b/database.sql @@ -2034,7 +2034,7 @@ CREATE TABLE IF NOT EXISTS `worker-ipc` ( -- VIEW application-view -- DROP VIEW IF EXISTS `application-view`; -CREATE VIEW `application-view` AS SELECT +CREATE VIEW `application-view` AS SELECT `application`.`id` AS `id`, `application-token`.`uid` AS `uid`, `application`.`name` AS `name`, @@ -2058,7 +2058,7 @@ CREATE VIEW `application-view` AS SELECT -- VIEW circle-member-view -- DROP VIEW IF EXISTS `circle-member-view`; -CREATE VIEW `circle-member-view` AS SELECT +CREATE VIEW `circle-member-view` AS SELECT `group_member`.`id` AS `id`, `group`.`uid` AS `uid`, `group_member`.`contact-id` AS `contact-id`, @@ -2089,7 +2089,7 @@ CREATE VIEW `circle-member-view` AS SELECT -- VIEW post-counts-view -- DROP VIEW IF EXISTS `post-counts-view`; -CREATE VIEW `post-counts-view` AS SELECT +CREATE VIEW `post-counts-view` AS SELECT `post-counts`.`uri-id` AS `uri-id`, `post-counts`.`vid` AS `vid`, `verb`.`name` AS `verb`, @@ -2103,7 +2103,7 @@ CREATE VIEW `post-counts-view` AS SELECT -- VIEW post-engagement-user-view -- DROP VIEW IF EXISTS `post-engagement-user-view`; -CREATE VIEW `post-engagement-user-view` AS SELECT +CREATE VIEW `post-engagement-user-view` AS SELECT `post-thread-user`.`uid` AS `uid`, `post-engagement`.`uri-id` AS `uri-id`, `post-engagement`.`owner-id` AS `owner-id`, @@ -2135,7 +2135,7 @@ CREATE VIEW `post-engagement-user-view` AS SELECT -- VIEW post-timeline-view -- DROP VIEW IF EXISTS `post-timeline-view`; -CREATE VIEW `post-timeline-view` AS SELECT +CREATE VIEW `post-timeline-view` AS SELECT `post-user`.`uid` AS `uid`, `post-user`.`uri-id` AS `uri-id`, `post-user`.`gravity` AS `gravity`, @@ -2180,7 +2180,7 @@ CREATE VIEW `post-timeline-view` AS SELECT -- VIEW post-timeline-origin-view -- DROP VIEW IF EXISTS `post-timeline-origin-view`; -CREATE VIEW `post-timeline-origin-view` AS SELECT +CREATE VIEW `post-timeline-origin-view` AS SELECT `post-origin`.`uid` AS `uid`, `post-origin`.`uri-id` AS `uri-id`, `post-origin`.`gravity` AS `gravity`, @@ -2226,7 +2226,7 @@ CREATE VIEW `post-timeline-origin-view` AS SELECT -- VIEW post-searchindex-user-view -- DROP VIEW IF EXISTS `post-searchindex-user-view`; -CREATE VIEW `post-searchindex-user-view` AS SELECT +CREATE VIEW `post-searchindex-user-view` AS SELECT `post-thread-user`.`uid` AS `uid`, `post-searchindex`.`uri-id` AS `uri-id`, `post-searchindex`.`owner-id` AS `owner-id`, @@ -2258,7 +2258,7 @@ CREATE VIEW `post-searchindex-user-view` AS SELECT -- VIEW post-origin-view -- DROP VIEW IF EXISTS `post-origin-view`; -CREATE VIEW `post-origin-view` AS SELECT +CREATE VIEW `post-origin-view` AS SELECT `post-origin`.`id` AS `id`, `post-origin`.`id` AS `post-user-id`, `post-origin`.`uid` AS `uid`, @@ -2446,7 +2446,7 @@ CREATE VIEW `post-origin-view` AS SELECT -- VIEW post-thread-origin-view -- DROP VIEW IF EXISTS `post-thread-origin-view`; -CREATE VIEW `post-thread-origin-view` AS SELECT +CREATE VIEW `post-thread-origin-view` AS SELECT `post-origin`.`id` AS `id`, `post-origin`.`id` AS `post-user-id`, `post-origin`.`uid` AS `uid`, @@ -2632,7 +2632,7 @@ CREATE VIEW `post-thread-origin-view` AS SELECT -- VIEW post-user-view -- DROP VIEW IF EXISTS `post-user-view`; -CREATE VIEW `post-user-view` AS SELECT +CREATE VIEW `post-user-view` AS SELECT `post-user`.`id` AS `id`, `post-user`.`id` AS `post-user-id`, `post-user`.`uid` AS `uid`, @@ -2819,7 +2819,7 @@ CREATE VIEW `post-user-view` AS SELECT -- VIEW post-thread-user-view -- DROP VIEW IF EXISTS `post-thread-user-view`; -CREATE VIEW `post-thread-user-view` AS SELECT +CREATE VIEW `post-thread-user-view` AS SELECT `post-user`.`id` AS `id`, `post-user`.`id` AS `post-user-id`, `post-thread-user`.`uid` AS `uid`, @@ -3004,7 +3004,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT -- VIEW post-view -- DROP VIEW IF EXISTS `post-view`; -CREATE VIEW `post-view` AS SELECT +CREATE VIEW `post-view` AS SELECT `item-uri`.`uri` AS `uri`, `post`.`uri-id` AS `uri-id`, `parent-item-uri`.`uri` AS `parent-uri`, @@ -3151,7 +3151,7 @@ CREATE VIEW `post-view` AS SELECT -- VIEW post-thread-view -- DROP VIEW IF EXISTS `post-thread-view`; -CREATE VIEW `post-thread-view` AS SELECT +CREATE VIEW `post-thread-view` AS SELECT `item-uri`.`uri` AS `uri`, `post-thread`.`uri-id` AS `uri-id`, `parent-item-uri`.`uri` AS `parent-uri`, @@ -3299,7 +3299,7 @@ CREATE VIEW `post-thread-view` AS SELECT -- VIEW category-view -- DROP VIEW IF EXISTS `category-view`; -CREATE VIEW `category-view` AS SELECT +CREATE VIEW `category-view` AS SELECT `post-category`.`uri-id` AS `uri-id`, `post-category`.`uid` AS `uid`, `post-category`.`type` AS `type`, @@ -3313,7 +3313,7 @@ CREATE VIEW `category-view` AS SELECT -- VIEW collection-view -- DROP VIEW IF EXISTS `collection-view`; -CREATE VIEW `collection-view` AS SELECT +CREATE VIEW `collection-view` AS SELECT `post-collection`.`uri-id` AS `uri-id`, `post-collection`.`type` AS `type`, `post-collection`.`author-id` AS `cid`, @@ -3334,7 +3334,7 @@ CREATE VIEW `collection-view` AS SELECT -- VIEW media-view -- DROP VIEW IF EXISTS `media-view`; -CREATE VIEW `media-view` AS SELECT +CREATE VIEW `media-view` AS SELECT `post-media`.`uri-id` AS `uri-id`, `post-media`.`type` AS `type`, `post`.`received` AS `received`, @@ -3352,7 +3352,7 @@ CREATE VIEW `media-view` AS SELECT -- VIEW tag-view -- DROP VIEW IF EXISTS `tag-view`; -CREATE VIEW `tag-view` AS SELECT +CREATE VIEW `tag-view` AS SELECT `post-tag`.`uri-id` AS `uri-id`, `post-tag`.`type` AS `type`, `post-tag`.`tid` AS `tid`, @@ -3368,7 +3368,7 @@ CREATE VIEW `tag-view` AS SELECT -- VIEW network-thread-view -- DROP VIEW IF EXISTS `network-thread-view`; -CREATE VIEW `network-thread-view` AS SELECT +CREATE VIEW `network-thread-view` AS SELECT `post-thread-user`.`uri-id` AS `uri-id`, `post-thread-user`.`post-user-id` AS `parent`, `post-thread-user`.`received` AS `received`, @@ -3396,7 +3396,7 @@ CREATE VIEW `network-thread-view` AS SELECT -- VIEW network-thread-circle-view -- DROP VIEW IF EXISTS `network-thread-circle-view`; -CREATE VIEW `network-thread-circle-view` AS SELECT +CREATE VIEW `network-thread-circle-view` AS SELECT `post-thread-user`.`uri-id` AS `uri-id`, `post-thread-user`.`post-user-id` AS `parent`, `post-thread-user`.`received` AS `received`, @@ -3424,7 +3424,7 @@ CREATE VIEW `network-thread-circle-view` AS SELECT -- VIEW owner-view -- DROP VIEW IF EXISTS `owner-view`; -CREATE VIEW `owner-view` AS SELECT +CREATE VIEW `owner-view` AS SELECT `contact`.`id` AS `id`, `contact`.`uid` AS `uid`, `contact`.`created` AS `created`, @@ -3552,7 +3552,7 @@ CREATE VIEW `owner-view` AS SELECT -- VIEW account-view -- DROP VIEW IF EXISTS `account-view`; -CREATE VIEW `account-view` AS SELECT +CREATE VIEW `account-view` AS SELECT `contact`.`id` AS `id`, `contact`.`url` AS `url`, `contact`.`nurl` AS `nurl`, @@ -3640,7 +3640,7 @@ CREATE VIEW `account-view` AS SELECT -- VIEW account-user-view -- DROP VIEW IF EXISTS `account-user-view`; -CREATE VIEW `account-user-view` AS SELECT +CREATE VIEW `account-user-view` AS SELECT `ucontact`.`id` AS `id`, `contact`.`id` AS `pid`, `ucontact`.`uid` AS `uid`, @@ -3746,7 +3746,7 @@ CREATE VIEW `account-user-view` AS SELECT -- VIEW pending-view -- DROP VIEW IF EXISTS `pending-view`; -CREATE VIEW `pending-view` AS SELECT +CREATE VIEW `pending-view` AS SELECT `register`.`id` AS `id`, `register`.`hash` AS `hash`, `register`.`created` AS `created`, @@ -3768,7 +3768,7 @@ CREATE VIEW `pending-view` AS SELECT -- VIEW tag-search-view -- DROP VIEW IF EXISTS `tag-search-view`; -CREATE VIEW `tag-search-view` AS SELECT +CREATE VIEW `tag-search-view` AS SELECT `post-tag`.`uri-id` AS `uri-id`, `post-user`.`uid` AS `uid`, `post-user`.`id` AS `iid`, @@ -3790,7 +3790,7 @@ CREATE VIEW `tag-search-view` AS SELECT -- VIEW workerqueue-view -- DROP VIEW IF EXISTS `workerqueue-view`; -CREATE VIEW `workerqueue-view` AS SELECT +CREATE VIEW `workerqueue-view` AS SELECT `process`.`pid` AS `pid`, `workerqueue`.`priority` AS `priority` FROM `process` @@ -3801,7 +3801,7 @@ CREATE VIEW `workerqueue-view` AS SELECT -- VIEW profile_field-view -- DROP VIEW IF EXISTS `profile_field-view`; -CREATE VIEW `profile_field-view` AS SELECT +CREATE VIEW `profile_field-view` AS SELECT `profile_field`.`id` AS `id`, `profile_field`.`uid` AS `uid`, `profile_field`.`label` AS `label`, @@ -3821,7 +3821,7 @@ CREATE VIEW `profile_field-view` AS SELECT -- VIEW diaspora-contact-view -- DROP VIEW IF EXISTS `diaspora-contact-view`; -CREATE VIEW `diaspora-contact-view` AS SELECT +CREATE VIEW `diaspora-contact-view` AS SELECT `diaspora-contact`.`uri-id` AS `uri-id`, `item-uri`.`uri` AS `url`, `item-uri`.`guid` AS `guid`, diff --git a/src/DI.php b/src/DI.php index a984bc934f..99d6e1012d 100644 --- a/src/DI.php +++ b/src/DI.php @@ -66,7 +66,7 @@ abstract class DI public static function setCompositeRootDependencyByHand() { $database = static::dba(); - $database->setDependency(static::config(), static::profiler(), static::logger()); + $database->setDependency(static::config(), static::profiler(), static::logger(), static::lock()); } /** diff --git a/src/Database/DBA.php b/src/Database/DBA.php index d5e5a7e59b..afbb5e130d 100644 --- a/src/Database/DBA.php +++ b/src/Database/DBA.php @@ -21,6 +21,7 @@ namespace Friendica\Database; +use Friendica\Core\Lock\Exception\LockPersistenceException; use Friendica\DI; use mysqli; use mysqli_result; @@ -823,6 +824,27 @@ class DBA return DI::dba()->optimizeTable($table); } + /** + * Acquire a lock to prevent a table optimization + * + * @return bool + * @throws LockPersistenceException + */ + public static function acquireOptimizeLock(): bool + { + return DI::dba()->acquireOptimizeLock(); + } + + /** + * Release the table optimization lock + * @return bool + * @throws LockPersistenceException + */ + public static function releaseOptimizeLock(): bool + { + return DI::dba()->releaseOptimizeLock(); + } + /** * Kill sleeping database processes */ diff --git a/src/Database/Database.php b/src/Database/Database.php index c296683d2b..2691495850 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -22,6 +22,8 @@ namespace Friendica\Database; use Friendica\Core\Config\Capability\IManageConfigValues; +use Friendica\Core\Lock\Capability\ICanLock; +use Friendica\Core\Lock\Exception\LockPersistenceException; use Friendica\Core\System; use Friendica\Database\Definition\DbaDefinition; use Friendica\Database\Definition\ViewDefinition; @@ -50,6 +52,8 @@ class Database const INSERT_UPDATE = 1; const INSERT_IGNORE = 2; + const LOCK_OPTIMIZE = 'database::optimize_tables'; + protected $connected = false; /** @@ -64,6 +68,11 @@ class Database * @var LoggerInterface */ protected $logger = null; + /** + * @var ICanLock + */ + protected $syslock = null; + protected $server_info = ''; /** @var PDO|mysqli */ protected $connection; @@ -106,11 +115,12 @@ class Database * * @todo Make this method obsolete - use a clean pattern instead ... */ - public function setDependency(IManageConfigValues $config, Profiler $profiler, LoggerInterface $logger) + public function setDependency(IManageConfigValues $config, Profiler $profiler, LoggerInterface $logger, ICanLock $lock) { $this->logger = $logger; $this->profiler = $profiler; $this->config = $config; + $this->syslock = $lock; } /** @@ -1755,7 +1765,42 @@ class Database */ public function optimizeTable(string $table): bool { - return $this->e("OPTIMIZE TABLE " . DBA::buildTableString([$table])) !== false; + if ($this->syslock->isLocked(self::LOCK_OPTIMIZE)) { + $this->logger->info('Optimization is locked'); + return false; + } + + if (!$this->acquireOptimizeLock()) { + return false; + } + + $result = $this->e("OPTIMIZE TABLE " . DBA::buildTableString([$table])) !== false; + + $this->releaseOptimizeLock(); + + return $result; + } + + /** + * Acquire a lock to prevent a table optimization + * + * @return bool + * @throws LockPersistenceException + */ + public function acquireOptimizeLock(): bool + { + return $this->syslock->acquire(self::LOCK_OPTIMIZE, 0); + } + + /** + * Release the table optimization lock + * + * @return bool + * @throws LockPersistenceException + */ + public function releaseOptimizeLock(): bool + { + return $this->syslock->release(self::LOCK_OPTIMIZE); } /** diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index 3ff58a08c4..3e4958c6a2 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -40,6 +40,11 @@ class ExpirePosts */ public static function execute() { + if (!DBA::acquireOptimizeLock()) { + Logger::warning('Lock could not be acquired'); + return; + } + self::deleteExpiredOriginPosts(); self::deleteOrphanedEntries(); @@ -52,6 +57,8 @@ class ExpirePosts self::addMissingEntries(); } + DBA::releaseOptimizeLock(); + // Set the expiry for origin posts Worker::add(Worker::PRIORITY_LOW, 'Expire'); @@ -69,15 +76,22 @@ class ExpirePosts Logger::notice('Delete expired posts'); // physically remove anything that has been deleted for more than two months $condition = ["`gravity` = ? AND `deleted` AND `changed` < ?", Item::GRAVITY_PARENT, DateTimeFormat::utc('now - 60 days')]; - $rows = Post::select(['guid', 'uri-id', 'uid'], $condition); - while ($row = Post::fetch($rows)) { - Logger::info('Delete expired item', ['uri-id' => $row['uri-id'], 'guid' => $row['guid']]); - Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); - Post\Origin::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); - } - DBA::close($rows); - - Logger::notice('Delete expired posts - done'); + $pass = 0; + do { + ++$pass; + $rows = Post::select(['guid', 'uri-id', 'uid'], $condition, ['limit' => 100]); + $affected_count = 0; + while ($row = Post::fetch($rows)) { + Logger::info('Delete expired item', ['pass' => $pass, 'uri-id' => $row['uri-id'], 'guid' => $row['guid']]); + Post\User::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); + $affected_count += DBA::affectedRows(); + Post\Origin::delete(['parent-uri-id' => $row['uri-id'], 'uid' => $row['uid']]); + $affected_count += DBA::affectedRows(); + } + DBA::close($rows); + DBA::commit(); + Logger::notice('Delete expired posts - done', ['pass' => $pass, 'rows' => $affected_count]); + } while ($affected_count); } /** @@ -106,6 +120,7 @@ class ExpirePosts $affected_count += DBA::affectedRows(); } DBA::close($uris); + DBA::commit(); Logger::notice('Orphaned entries deleted', ['table' => $table, 'rows' => $affected_count]); } Logger::notice('Delete orphaned entries - done'); @@ -172,14 +187,18 @@ class ExpirePosts { // We have to avoid deleting newly created "item-uri" entries. // So we fetch a post that had been stored yesterday and only delete older ones. - $item = Post::selectFirstThread(['uri-id'], ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - 1 day')], - ['order' => ['received' => true]]); + $item = Post::selectFirstThread( + ['uri-id'], + ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - 1 day')], + ['order' => ['received' => true]] + ); if (empty($item['uri-id'])) { Logger::warning('No item with uri-id found - we better quit here'); return; } Logger::notice('Start collecting orphaned URI-ID', ['last-id' => $item['uri-id']]); - $uris = DBA::select('item-uri', ['id'], ["`id` < ? + $condition = [ + "`id` < ? AND NOT EXISTS(SELECT `uri-id` FROM `post-user` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`) @@ -195,18 +214,24 @@ class ExpirePosts AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `inbox-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`) - AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]); - - Logger::notice('Start deleting orphaned URI-ID', ['last-id' => $item['uri-id']]); - $affected_count = 0; - while ($rows = DBA::toArray($uris, false, 100)) { - $ids = array_column($rows, 'id'); - DBA::delete('item-uri', ['id' => $ids]); - $affected_count += DBA::affectedRows(); - Logger::info('Deleted', ['rows' => $affected_count]); - } - DBA::close($uris); - Logger::notice('Orphaned URI-ID entries removed', ['rows' => $affected_count]); + AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id'] + ]; + $pass = 0; + do { + ++$pass; + $uris = DBA::select('item-uri', ['id'], $condition, ['limit' => 1000]); + Logger::notice('Start deleting orphaned URI-ID', ['pass' => $pass, 'last-id' => $item['uri-id']]); + $affected_count = 0; + while ($rows = DBA::toArray($uris, false, 100)) { + $ids = array_column($rows, 'id'); + DBA::delete('item-uri', ['id' => $ids]); + $affected_count += DBA::affectedRows(); + Logger::info('Deleted', ['pass' => $pass, 'rows' => $affected_count]); + } + DBA::close($uris); + DBA::commit(); + Logger::notice('Orphaned URI-ID entries removed', ['pass' => $pass, 'rows' => $affected_count]); + } while ($affected_count); } /** @@ -227,55 +252,68 @@ class ExpirePosts if (!empty($expire_days)) { Logger::notice('Start collecting expired threads', ['expiry_days' => $expire_days]); - $uris = DBA::select('item-uri', ['id'], ["`id` IN - (SELECT `uri-id` FROM `post-thread` WHERE `received` < ? - AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user` - WHERE (`mention` OR `starred` OR `wall`) AND `uri-id` = `post-thread`.`uri-id`) - AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category` - WHERE `uri-id` = `post-thread`.`uri-id`) - AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-collection` - WHERE `uri-id` = `post-thread`.`uri-id`) - AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media` - WHERE `uri-id` = `post-thread`.`uri-id`) - AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts` - WHERE `parent-uri-id` = `post-thread`.`uri-id`) - AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` - WHERE (`origin` OR `event-id` != 0 OR `post-type` = ?) AND `parent-uri-id` = `post-thread`.`uri-id`) - AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-content` - WHERE `resource-id` != 0 AND `uri-id` = `post-thread`.`uri-id`))", - DateTimeFormat::utc('now - ' . (int)$expire_days . ' days'), Item::PT_PERSONAL_NOTE]); + $condition = [ + "`id` IN (SELECT `uri-id` FROM `post-thread` WHERE `received` < ? + AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-thread-user` + WHERE (`mention` OR `starred` OR `wall`) AND `uri-id` = `post-thread`.`uri-id`) + AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-category` + WHERE `uri-id` = `post-thread`.`uri-id`) + AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-collection` + WHERE `uri-id` = `post-thread`.`uri-id`) + AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-media` + WHERE `uri-id` = `post-thread`.`uri-id`) + AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` INNER JOIN `contact` ON `contact`.`id` = `contact-id` AND `notify_new_posts` + WHERE `parent-uri-id` = `post-thread`.`uri-id`) + AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` + WHERE (`origin` OR `event-id` != 0 OR `post-type` = ?) AND `parent-uri-id` = `post-thread`.`uri-id`) + AND NOT `uri-id` IN (SELECT `uri-id` FROM `post-content` + WHERE `resource-id` != 0 AND `uri-id` = `post-thread`.`uri-id`))", + DateTimeFormat::utc('now - ' . (int)$expire_days . ' days'), Item::PT_PERSONAL_NOTE + ]; + $pass = 0; + do { + ++$pass; + $uris = DBA::select('item-uri', ['id'], $condition, ['limit' => 100]); - Logger::notice('Start deleting expired threads'); - $affected_count = 0; - while ($rows = DBA::toArray($uris, false, 100)) { - $ids = array_column($rows, 'id'); - DBA::delete('item-uri', ['id' => $ids]); - $affected_count += DBA::affectedRows(); - } - DBA::close($uris); - - Logger::notice('Deleted expired threads', ['rows' => $affected_count]); + Logger::notice('Start deleting expired threads', ['pass' => $pass]); + $affected_count = 0; + while ($rows = DBA::toArray($uris, false, 100)) { + $ids = array_column($rows, 'id'); + DBA::delete('item-uri', ['id' => $ids]); + $affected_count += DBA::affectedRows(); + } + DBA::close($uris); + DBA::commit(); + Logger::notice('Deleted expired threads', ['pass' => $pass, 'rows' => $affected_count]); + } while ($affected_count); } if (!empty($expire_days_unclaimed)) { Logger::notice('Start collecting unclaimed public items', ['expiry_days' => $expire_days_unclaimed]); - $uris = DBA::select('item-uri', ['id'], ["`id` IN - (SELECT `uri-id` FROM `post-user` WHERE `gravity` = ? AND `uid` = ? AND `received` < ? - AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` != ? - AND `i`.`parent-uri-id` = `post-user`.`uri-id`) - AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` = ? - AND `i`.`parent-uri-id` = `post-user`.`uri-id` AND `i`.`received` > ?))", - Item::GRAVITY_PARENT, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days'), 0, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days')]); + $condition = [ + "`id` IN (SELECT `uri-id` FROM `post-user` WHERE `gravity` = ? AND `uid` = ? AND `received` < ? + AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` != ? + AND `i`.`parent-uri-id` = `post-user`.`uri-id`) + AND NOT `uri-id` IN (SELECT `parent-uri-id` FROM `post-user` AS `i` WHERE `i`.`uid` = ? + AND `i`.`parent-uri-id` = `post-user`.`uri-id` AND `i`.`received` > ?))", + Item::GRAVITY_PARENT, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days'), 0, 0, DateTimeFormat::utc('now - ' . (int)$expire_days_unclaimed . ' days') + ]; + $pass = 0; + do { + ++$pass; + $uris = DBA::select('item-uri', ['id'], $condition, ['limit' => 100]); - Logger::notice('Start deleting unclaimed public items'); - $affected_count = 0; - while ($rows = DBA::toArray($uris, false, 100)) { - $ids = array_column($rows, 'id'); - DBA::delete('item-uri', ['id' => $ids]); - $affected_count += DBA::affectedRows(); - } - DBA::close($uris); - Logger::notice('Deleted unclaimed public items', ['rows' => $affected_count]); + Logger::notice('Start deleting unclaimed public items', ['pass' => $pass]); + $affected_count = 0; + while ($rows = DBA::toArray($uris, false, 100)) { + $ids = array_column($rows, 'id'); + DBA::delete('item-uri', ['id' => $ids]); + $affected_count += DBA::affectedRows(); + } + DBA::close($uris); + DBA::commit(); + Logger::notice('Deleted unclaimed public items', ['pass' => $pass, 'rows' => $affected_count]); + } while ($affected_count); } } }