From b136c5f32bcadda873bbe14791dc7cce3d2d9833 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 16 Aug 2020 08:39:04 +0000 Subject: [PATCH] Split the CXronjobs class into separate classes --- src/Model/User.php | 2 +- src/Worker/ClearCache.php | 99 ++++++++++ src/Worker/Cron.php | 14 +- src/Worker/CronJobs.php | 266 -------------------------- src/Worker/ExpireAndRemoveUsers.php | 58 ++++++ src/Worker/MoveStorage.php | 43 +++++ src/Worker/NodeInfo.php | 41 ++++ src/Worker/PostUpdate.php | 32 ++++ src/Worker/RepairDatabase.php | 69 +++++++ src/Worker/UpdateContactBirthdays.php | 32 ++++ src/Worker/UpdatePhotoAlbums.php | 40 ++++ 11 files changed, 422 insertions(+), 274 deletions(-) create mode 100644 src/Worker/ClearCache.php delete mode 100644 src/Worker/CronJobs.php create mode 100644 src/Worker/ExpireAndRemoveUsers.php create mode 100644 src/Worker/MoveStorage.php create mode 100644 src/Worker/NodeInfo.php create mode 100644 src/Worker/PostUpdate.php create mode 100644 src/Worker/RepairDatabase.php create mode 100644 src/Worker/UpdateContactBirthdays.php create mode 100644 src/Worker/UpdatePhotoAlbums.php diff --git a/src/Model/User.php b/src/Model/User.php index 46f0776b4e..f7c683b62b 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -1180,7 +1180,7 @@ class User // unique), so it cannot be re-registered in the future. DBA::insert('userd', ['username' => $user['nickname']]); - // The user and related data will be deleted in Friendica\Worker\CronJobs::expireAndRemoveUsers() + // The user and related data will be deleted in Friendica\Worker\ExpireAndRemoveUsers DBA::update('user', ['account_removed' => true, 'account_expires_on' => DateTimeFormat::utc('now + 7 day')], ['uid' => $uid]); Worker::add(PRIORITY_HIGH, 'Notifier', Delivery::REMOVAL, $uid); diff --git a/src/Worker/ClearCache.php b/src/Worker/ClearCache.php new file mode 100644 index 0000000000..692a462a77 --- /dev/null +++ b/src/Worker/ClearCache.php @@ -0,0 +1,99 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Core\Logger; +use Friendica\Database\DBA; +use Friendica\DI; +use Friendica\Model\Photo; +use Friendica\Util\Proxy as ProxyUtils; + +/** + * Clear cache entries + */ +class ClearCache +{ + public static function execute() + { + $a = DI::app(); + $last = DI::config()->get('system', 'cache_last_cleared'); + + if ($last) { + $next = $last + (3600); // Once per hour + $clear_cache = ($next <= time()); + } else { + $clear_cache = true; + } + + if (!$clear_cache) { + return; + } + + // clear old cache + DI::cache()->clear(); + + // clear old item cache files + clear_cache(); + + // clear cache for photos + clear_cache($a->getBasePath(), $a->getBasePath() . "/photo"); + + // clear smarty cache + clear_cache($a->getBasePath() . "/view/smarty3/compiled", $a->getBasePath() . "/view/smarty3/compiled"); + + // clear cache for image proxy + if (!DI::config()->get("system", "proxy_disabled")) { + clear_cache($a->getBasePath(), $a->getBasePath() . "/proxy"); + + $cachetime = DI::config()->get('system', 'proxy_cache_time'); + + if (!$cachetime) { + $cachetime = ProxyUtils::DEFAULT_TIME; + } + + $condition = ['`uid` = 0 AND `resource-id` LIKE "pic:%" AND `created` < NOW() - INTERVAL ? SECOND', $cachetime]; + Photo::delete($condition); + } + + // Delete the cached OEmbed entries that are older than three month + DBA::delete('oembed', ["`created` < NOW() - INTERVAL 3 MONTH"]); + + // Delete the cached "parse_url" entries that are older than three month + DBA::delete('parsed_url', ["`created` < NOW() - INTERVAL 3 MONTH"]); + + if (DI::config()->get('system', 'optimize_tables')) { + Logger::info('Optimize start'); + DBA::e("OPTIMIZE TABLE `auth_codes`"); + DBA::e("OPTIMIZE TABLE `cache`"); + DBA::e("OPTIMIZE TABLE `challenge`"); + DBA::e("OPTIMIZE TABLE `locks`"); + DBA::e("OPTIMIZE TABLE `oembed`"); + DBA::e("OPTIMIZE TABLE `parsed_url`"); + DBA::e("OPTIMIZE TABLE `profile_check`"); + DBA::e("OPTIMIZE TABLE `session`"); + DBA::e("OPTIMIZE TABLE `tokens`"); + Logger::info('Optimize finished'); + } + + DI::config()->set('system', 'cache_last_cleared', time()); + } +} diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 9ce00fb936..ecfd2e7ddc 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -58,16 +58,16 @@ class Cron Worker::add(PRIORITY_LOW, 'UpdateServerDirectories'); // Expire and remove user entries - Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users"); + Worker::add(PRIORITY_MEDIUM, 'ExpireAndRemoveUsers'); // Call possible post update functions - Worker::add(PRIORITY_LOW, "CronJobs", "post_update"); + Worker::add(PRIORITY_LOW, 'PostUpdate'); // Clear cache entries - Worker::add(PRIORITY_LOW, "CronJobs", "clear_cache"); + Worker::add(PRIORITY_LOW, 'ClearCache'); // Repair entries in the database - Worker::add(PRIORITY_LOW, "CronJobs", "repair_database"); + Worker::add(PRIORITY_LOW, 'RepairDatabase'); // once daily run birthday_updates and then expire in background $d1 = DI::config()->get('system', 'last_expire_day'); @@ -76,12 +76,12 @@ class Cron // Daily cron calls if ($d2 != intval($d1)) { - Worker::add(PRIORITY_LOW, "CronJobs", "update_contact_birthdays"); + Worker::add(PRIORITY_LOW, 'UpdateContactBirthdays'); - Worker::add(PRIORITY_LOW, "CronJobs", "update_photo_albums"); + Worker::add(PRIORITY_LOW, 'UpdatePhotoAlbums'); // update nodeinfo data - Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo"); + Worker::add(PRIORITY_LOW, 'NodeInfo'); Worker::add(PRIORITY_LOW, 'UpdateGServers'); diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php deleted file mode 100644 index 385109ae20..0000000000 --- a/src/Worker/CronJobs.php +++ /dev/null @@ -1,266 +0,0 @@ -. - * - */ - -namespace Friendica\Worker; - -use Friendica\App; -use Friendica\Core\Logger; -use Friendica\Core\Protocol; -use Friendica\Core\Worker; -use Friendica\Database\DBA; -use Friendica\Database\PostUpdate; -use Friendica\DI; -use Friendica\Model\Contact; -use Friendica\Model\Nodeinfo; -use Friendica\Model\Photo; -use Friendica\Model\User; -use Friendica\Util\Proxy as ProxyUtils; -use Friendica\Util\Strings; - -class CronJobs -{ - public static function execute($command = '') - { - $a = DI::app(); - - // No parameter set? So return - if ($command == '') { - return; - } - - Logger::log("Starting cronjob " . $command, Logger::DEBUG); - - switch($command) { - case 'post_update': - PostUpdate::update(); - break; - - case 'nodeinfo': - Logger::info('cron_start'); - Nodeinfo::update(); - // Now trying to register - $url = 'http://the-federation.info/register/' . DI::baseUrl()->getHostname(); - Logger::debug('Check registering url', ['url' => $url]); - $ret = DI::httpRequest()->fetch($url); - Logger::debug('Check registering answer', ['answer' => $ret]); - Logger::info('cron_end'); - break; - - case 'expire_and_remove_users': - self::expireAndRemoveUsers(); - break; - - case 'update_contact_birthdays': - Contact::updateBirthdays(); - break; - - case 'update_photo_albums': - self::updatePhotoAlbums(); - break; - - case 'clear_cache': - self::clearCache($a); - break; - - case 'repair_database': - self::repairDatabase(); - break; - - case 'move_storage': - self::moveStorage(); - break; - - default: - Logger::log("Cronjob " . $command . " is unknown.", Logger::DEBUG); - } - - return; - } - - /** - * Update the cached values for the number of photo albums per user - */ - private static function updatePhotoAlbums() - { - $r = q("SELECT `uid` FROM `user` WHERE NOT `account_expired` AND NOT `account_removed`"); - if (!DBA::isResult($r)) { - return; - } - - foreach ($r as $user) { - Photo::clearAlbumCache($user['uid']); - } - } - - /** - * Expire and remove user entries - */ - private static function expireAndRemoveUsers() - { - // expire any expired regular accounts. Don't expire forums. - $condition = ["NOT `account_expired` AND `account_expires_on` > ? AND `account_expires_on` < UTC_TIMESTAMP() AND `page-flags` = 0", DBA::NULL_DATETIME]; - DBA::update('user', ['account_expired' => true], $condition); - - // Remove any freshly expired account - $users = DBA::select('user', ['uid'], ['account_expired' => true, 'account_removed' => false]); - while ($user = DBA::fetch($users)) { - User::remove($user['uid']); - } - DBA::close($users); - - // delete user records for recently removed accounts - $users = DBA::select('user', ['uid'], ["`account_removed` AND `account_expires_on` < UTC_TIMESTAMP() "]); - while ($user = DBA::fetch($users)) { - // Delete the contacts of this user - $self = DBA::selectFirst('contact', ['nurl'], ['self' => true, 'uid' => $user['uid']]); - if (DBA::isResult($self)) { - DBA::delete('contact', ['nurl' => $self['nurl'], 'self' => false]); - } - - DBA::delete('user', ['uid' => $user['uid']]); - } - DBA::close($users); - } - - /** - * Clear cache entries - * - * @param App $a - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - private static function clearCache(App $a) - { - $last = DI::config()->get('system', 'cache_last_cleared'); - - if ($last) { - $next = $last + (3600); // Once per hour - $clear_cache = ($next <= time()); - } else { - $clear_cache = true; - } - - if (!$clear_cache) { - return; - } - - // clear old cache - DI::cache()->clear(); - - // clear old item cache files - clear_cache(); - - // clear cache for photos - clear_cache($a->getBasePath(), $a->getBasePath() . "/photo"); - - // clear smarty cache - clear_cache($a->getBasePath() . "/view/smarty3/compiled", $a->getBasePath() . "/view/smarty3/compiled"); - - // clear cache for image proxy - if (!DI::config()->get("system", "proxy_disabled")) { - clear_cache($a->getBasePath(), $a->getBasePath() . "/proxy"); - - $cachetime = DI::config()->get('system', 'proxy_cache_time'); - - if (!$cachetime) { - $cachetime = ProxyUtils::DEFAULT_TIME; - } - - $condition = ['`uid` = 0 AND `resource-id` LIKE "pic:%" AND `created` < NOW() - INTERVAL ? SECOND', $cachetime]; - Photo::delete($condition); - } - - // Delete the cached OEmbed entries that are older than three month - DBA::delete('oembed', ["`created` < NOW() - INTERVAL 3 MONTH"]); - - // Delete the cached "parse_url" entries that are older than three month - DBA::delete('parsed_url', ["`created` < NOW() - INTERVAL 3 MONTH"]); - - if (DI::config()->get('system', 'optimize_tables')) { - Logger::info('Optimize start'); - DBA::e("OPTIMIZE TABLE `auth_codes`"); - DBA::e("OPTIMIZE TABLE `cache`"); - DBA::e("OPTIMIZE TABLE `challenge`"); - DBA::e("OPTIMIZE TABLE `locks`"); - DBA::e("OPTIMIZE TABLE `oembed`"); - DBA::e("OPTIMIZE TABLE `parsed_url`"); - DBA::e("OPTIMIZE TABLE `profile_check`"); - DBA::e("OPTIMIZE TABLE `session`"); - DBA::e("OPTIMIZE TABLE `tokens`"); - Logger::info('Optimize finished'); - } - - DI::config()->set('system', 'cache_last_cleared', time()); - } - - /** - * Do some repairs in database entries - * - */ - private static function repairDatabase() - { - // Sometimes there seem to be issues where the "self" contact vanishes. - // We haven't found the origin of the problem by now. - $r = q("SELECT `uid` FROM `user` WHERE NOT EXISTS (SELECT `uid` FROM `contact` WHERE `contact`.`uid` = `user`.`uid` AND `contact`.`self`)"); - if (DBA::isResult($r)) { - foreach ($r AS $user) { - Logger::log('Create missing self contact for user ' . $user['uid']); - Contact::createSelfFromUserId($user['uid']); - } - } - - // There was an issue where the nick vanishes from the contact table - q("UPDATE `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` SET `nick` = `nickname` WHERE `self` AND `nick`=''"); - - /// @todo - /// - remove thread entries without item - /// - remove sign entries without item - /// - remove children when parent got lost - /// - set contact-id in item when not present - - // Add intro entries for pending contacts - // We don't do this for DFRN entries since such revived contact requests seem to mostly fail. - $pending_contacts = DBA::p("SELECT `uid`, `id`, `url`, `network`, `created` FROM `contact` - WHERE `pending` AND `rel` IN (?, ?) AND `network` != ? - AND NOT EXISTS (SELECT `id` FROM `intro` WHERE `contact-id` = `contact`.`id`)", - 0, Contact::FOLLOWER, Protocol::DFRN); - while ($contact = DBA::fetch($pending_contacts)) { - DBA::insert('intro', ['uid' => $contact['uid'], 'contact-id' => $contact['id'], 'blocked' => false, - 'hash' => Strings::getRandomHex(), 'datetime' => $contact['created']]); - } - DBA::close($pending_contacts); - } - - /** - * Moves up to 5000 attachments and photos to the current storage system. - * Self-replicates if legacy items have been found and moved. - * - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - private static function moveStorage() - { - $current = DI::storage(); - $moved = DI::storageManager()->move($current); - - if ($moved) { - Worker::add(PRIORITY_LOW, "CronJobs", "move_storage"); - } - } -} diff --git a/src/Worker/ExpireAndRemoveUsers.php b/src/Worker/ExpireAndRemoveUsers.php new file mode 100644 index 0000000000..1fda31a179 --- /dev/null +++ b/src/Worker/ExpireAndRemoveUsers.php @@ -0,0 +1,58 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Database\DBA; +use Friendica\Model\User; + +/** + * Expire and remove user entries + */ +class ExpireAndRemoveUsers +{ + public static function execute() + { + // expire any expired regular accounts. Don't expire forums. + $condition = ["NOT `account_expired` AND `account_expires_on` > ? AND `account_expires_on` < UTC_TIMESTAMP() AND `page-flags` = 0", DBA::NULL_DATETIME]; + DBA::update('user', ['account_expired' => true], $condition); + + // Remove any freshly expired account + $users = DBA::select('user', ['uid'], ['account_expired' => true, 'account_removed' => false]); + while ($user = DBA::fetch($users)) { + User::remove($user['uid']); + } + DBA::close($users); + + // delete user records for recently removed accounts + $users = DBA::select('user', ['uid'], ["`account_removed` AND `account_expires_on` < UTC_TIMESTAMP() "]); + while ($user = DBA::fetch($users)) { + // Delete the contacts of this user + $self = DBA::selectFirst('contact', ['nurl'], ['self' => true, 'uid' => $user['uid']]); + if (DBA::isResult($self)) { + DBA::delete('contact', ['nurl' => $self['nurl'], 'self' => false]); + } + + DBA::delete('user', ['uid' => $user['uid']]); + } + DBA::close($users); + } +} diff --git a/src/Worker/MoveStorage.php b/src/Worker/MoveStorage.php new file mode 100644 index 0000000000..1ffb3c275d --- /dev/null +++ b/src/Worker/MoveStorage.php @@ -0,0 +1,43 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Core\Worker; +use Friendica\DI; + +/** + * Moves up to 5000 attachments and photos to the current storage system. + * Self-replicates if legacy items have been found and moved. + * + */ +class MoveStorage +{ + public static function execute() + { + $current = DI::storage(); + $moved = DI::storageManager()->move($current); + + if ($moved) { + Worker::add(PRIORITY_LOW, 'MoveStorage'); + } + } +} diff --git a/src/Worker/NodeInfo.php b/src/Worker/NodeInfo.php new file mode 100644 index 0000000000..60ed9e941a --- /dev/null +++ b/src/Worker/NodeInfo.php @@ -0,0 +1,41 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Core\Logger; +use Friendica\DI; +use Friendica\Model\Nodeinfo as ModelNodeInfo; + +class NodeInfo +{ + public static function execute() + { + Logger::info('start'); + ModelNodeInfo::update(); + // Now trying to register + $url = 'http://the-federation.info/register/' . DI::baseUrl()->getHostname(); + Logger::debug('Check registering url', ['url' => $url]); + $ret = DI::httpRequest()->fetch($url); + Logger::debug('Check registering answer', ['answer' => $ret]); + Logger::info('end'); + } +} diff --git a/src/Worker/PostUpdate.php b/src/Worker/PostUpdate.php new file mode 100644 index 0000000000..56e12da9ea --- /dev/null +++ b/src/Worker/PostUpdate.php @@ -0,0 +1,32 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Database\PostUpdate as DatabasePostUpdate; + +class PostUpdate +{ + public static function execute() + { + DatabasePostUpdate::update(); + } +} diff --git a/src/Worker/RepairDatabase.php b/src/Worker/RepairDatabase.php new file mode 100644 index 0000000000..9d951dadc6 --- /dev/null +++ b/src/Worker/RepairDatabase.php @@ -0,0 +1,69 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Core\Logger; +use Friendica\Core\Protocol; +use Friendica\Database\DBA; +use Friendica\Model\Contact; +use Friendica\Util\Strings; + +/** + * Do some repairs in database entries + * + */ +class RepairDatabase +{ + public static function execute() + { + // Sometimes there seem to be issues where the "self" contact vanishes. + // We haven't found the origin of the problem by now. + + $users = DBA::select('user', ['uid'], ["NOT EXISTS (SELECT `uid` FROM `contact` WHERE `contact`.`uid` = `user`.`uid` AND `contact`.`self`)"]); + while ($user = DBA::fetch($users)) { + Logger::notice('Create missing self contact', ['user'=> $user['uid']]); + Contact::createSelfFromUserId($user['uid']); + } + DBA::close($users); + + // There was an issue where the nick vanishes from the contact table + DBA::e("UPDATE `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` SET `nick` = `nickname` WHERE `self` AND `nick`=''"); + + /// @todo + /// - remove thread entries without item + /// - remove sign entries without item + /// - remove children when parent got lost + /// - set contact-id in item when not present + + // Add intro entries for pending contacts + // We don't do this for DFRN entries since such revived contact requests seem to mostly fail. + $pending_contacts = DBA::p("SELECT `uid`, `id`, `url`, `network`, `created` FROM `contact` + WHERE `pending` AND `rel` IN (?, ?) AND `network` != ? + AND NOT EXISTS (SELECT `id` FROM `intro` WHERE `contact-id` = `contact`.`id`)", + 0, Contact::FOLLOWER, Protocol::DFRN); + while ($contact = DBA::fetch($pending_contacts)) { + DBA::insert('intro', ['uid' => $contact['uid'], 'contact-id' => $contact['id'], 'blocked' => false, + 'hash' => Strings::getRandomHex(), 'datetime' => $contact['created']]); + } + DBA::close($pending_contacts); + } +} diff --git a/src/Worker/UpdateContactBirthdays.php b/src/Worker/UpdateContactBirthdays.php new file mode 100644 index 0000000000..961c598289 --- /dev/null +++ b/src/Worker/UpdateContactBirthdays.php @@ -0,0 +1,32 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Model\Contact; + +class UpdateContactBirthdays +{ + public static function execute() + { + Contact::updateBirthdays(); + } +} diff --git a/src/Worker/UpdatePhotoAlbums.php b/src/Worker/UpdatePhotoAlbums.php new file mode 100644 index 0000000000..4b4c0bf1b3 --- /dev/null +++ b/src/Worker/UpdatePhotoAlbums.php @@ -0,0 +1,40 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Database\DBA; +use Friendica\Model\Photo; + +/** + * Update the cached values for the number of photo albums per user + */ +class UpdatePhotoAlbums +{ + public static function execute() + { + $users = DBA::select('user', ['uid'], ['account_expired' => false, 'account_removed' => false]); + while ($user = DBA::fetch($users)) { + Photo::clearAlbumCache($user['uid']); + } + DBA::close($users); + } +}