Merge pull request #14315 from annando/stats

New endpoint for monitoring services

src/Core/Worker.php

@@ -235,7 +235,7 @@ class Worker
 	 * @return integer Number of deferred entries in the worker queue
 	 * @throws \Exception
 	 */
-	private static function deferredEntries(): int
+	public static function deferredEntries(): int
 	{
 		$stamp = (float)microtime(true);
 		$count = DBA::count('workerqueue', ["NOT `done` AND `pid` = 0 AND `retrial` > ?", 0]);
@@ -250,7 +250,7 @@ class Worker
 	 * @return integer Number of non executed entries in the worker queue
 	 * @throws \Exception
 	 */
-	private static function totalEntries(): int
+	public static function totalEntries(): int
 	{
 		$stamp = (float)microtime(true);
 		$count = DBA::count('workerqueue', ['done' => false, 'pid' => 0]);
@@ -862,7 +862,7 @@ class Worker
 	 * @return integer Number of active worker processes
 	 * @throws \Exception
 	 */
-	private static function activeWorkers(): int
+	public static function activeWorkers(): int
 	{
 		$stamp = (float)microtime(true);
 		$count = DI::process()->countCommand('Worker.php');

src/Model/Item.php

@@ -421,6 +421,14 @@ class Item
 
 		// Is it our comment and/or our thread?
 		if (($item['origin'] || $parent['origin']) && ($item['uid'] != 0)) {
+			if ($item['origin'] && $item['gravity'] == self::GRAVITY_PARENT) {
+				$posts = DI::keyValue()->get('nodeinfo_local_posts') ?? 0;
+				DI::keyValue()->set('nodeinfo_local_posts', $posts - 1);
+			} elseif ($item['origin'] && $item['gravity'] == self::GRAVITY_COMMENT) {
+				$comments = DI::keyValue()->get('nodeinfo_local_comments') ?? 0;
+				DI::keyValue()->set('nodeinfo_local_comments', $comments - 1);
+			}
+	
 			// When we delete the original post we will delete all existing copies on the server as well
 			self::markForDeletion(['uri-id' => $item['uri-id'], 'deleted' => false], $priority);
 
@@ -1350,6 +1358,14 @@ class Item
 			return 0;
 		}
 
+		if ($posted_item['origin'] && $posted_item['gravity'] == self::GRAVITY_PARENT) {
+			$posts = DI::keyValue()->get('nodeinfo_local_posts') ?? 0;
+			DI::keyValue()->set('nodeinfo_local_posts', $posts + 1);
+		} elseif ($posted_item['origin'] && $posted_item['gravity'] == self::GRAVITY_COMMENT) {
+			$comments = DI::keyValue()->get('nodeinfo_local_comments') ?? 0;
+			DI::keyValue()->set('nodeinfo_local_comments', $comments + 1);
+		}
+
 		Post\Origin::insert($posted_item);
 
 		// update the commented timestamp on the parent

src/Model/Nodeinfo.php

@@ -53,6 +53,8 @@ class Nodeinfo
 			return;
 		}
 
+		$logger->info('User statistics - start');
+
 		$userStats = User::getStatistics();
 
 		DI::keyValue()->set('nodeinfo_total_users', $userStats['total_users']);
@@ -60,14 +62,14 @@ class Nodeinfo
 		DI::keyValue()->set('nodeinfo_active_users_monthly', $userStats['active_users_monthly']);
 		DI::keyValue()->set('nodeinfo_active_users_weekly', $userStats['active_users_weekly']);
 
-		$logger->info('user statistics', $userStats);
+		$logger->info('user statistics - done', $userStats);
 
 		$posts = DBA::count('post-thread', ["`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE NOT `deleted` AND `origin`)"]);
 		$comments = DBA::count('post', ["NOT `deleted` AND `gravity` = ? AND `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `origin`)", Item::GRAVITY_COMMENT]);
 		DI::keyValue()->set('nodeinfo_local_posts', $posts);
 		DI::keyValue()->set('nodeinfo_local_comments', $comments);
 
-		$logger->info('User activity', ['posts' => $posts, 'comments' => $comments]);
+		$logger->info('Post statistics - done', ['posts' => $posts, 'comments' => $comments]);
 	}
 
 	/**

src/Module/Stats.php

@@ -0,0 +1,144 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2024, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Module;
+
+use Friendica\App;
+use Friendica\BaseModule;
+use Friendica\Core\Config\Capability\IManageConfigValues;
+use Friendica\Core\KeyValueStorage\Capability\IManageKeyValuePairs;
+use Friendica\Core\L10n;
+use Friendica\Core\Worker;
+use Friendica\Database\Database;
+use Friendica\Util\DateTimeFormat;
+use Friendica\Util\Profiler;
+use Psr\Log\LoggerInterface;
+use Friendica\Network\HTTPException;
+
+class Stats extends BaseModule
+{
+	/** @var IManageConfigValues */
+	protected $config;
+	/** @var Database */
+	protected $dba;
+	/** @var LoggerInterface */
+	protected $logger;
+	/** @var IManageKeyValuePairs */
+	protected $keyValue;
+
+	public function __construct(L10n $l10n, App\BaseURL $baseUrl, App\Arguments $args, LoggerInterface $logger, Profiler $profiler, IManageConfigValues $config, IManageKeyValuePairs $keyValue, Database $dba, Response $response, array $server, array $parameters = [])
+	{
+		parent::__construct($l10n, $baseUrl, $args, $logger, $profiler, $response, $server, $parameters);
+
+		$this->config   = $config;
+		$this->keyValue = $keyValue;
+		$this->dba      = $dba;
+	}
+
+	protected function content(array $request = []): string
+	{
+		if (!$this->isAllowed($request)) {
+			throw new HTTPException\NotFoundException($this->l10n->t('Page not found.'));
+		}
+		return '';
+	}
+
+	protected function rawContent(array $request = [])
+	{
+		if (!$this->isAllowed($request)) {
+			return;
+		}
+
+		$statistics = [
+			'worker' => [
+				'lastExecution' => [
+					'datetime'  => DateTimeFormat::utc($this->keyValue->get('last_worker_execution'), DateTimeFormat::JSON),
+					'timestamp' => strtotime($this->keyValue->get('last_worker_execution')),
+				],
+				'jpm'           => [
+					1 => $this->dba->count('workerqueue', ["`done` AND `executed` > ?", DateTimeFormat::utc('now - 1 minute')]),
+					3 => round($this->dba->count('workerqueue', ["`done` AND `executed` > ?", DateTimeFormat::utc('now - 3 minute')]) / 3),
+					5 => round($this->dba->count('workerqueue', ["`done` AND `executed` > ?", DateTimeFormat::utc('now - 5 minute')]) / 5),
+				],
+				'active'        => [],
+				'deferred'      => [],
+				'total'         => [],
+			],
+			'users' => [
+				'total'          => intval($this->keyValue->get('nodeinfo_total_users')),
+				'activeHalfyear' => intval($this->keyValue->get('nodeinfo_active_users_halfyear')),
+				'activeMonth'    => intval($this->keyValue->get('nodeinfo_active_users_monthly')),
+				'activeWeek'     => intval($this->keyValue->get('nodeinfo_active_users_weekly')),
+			],
+			'usage' => [
+				'localPosts'    => intval($this->keyValue->get('nodeinfo_local_posts')),
+				'localComments' => intval($this->keyValue->get('nodeinfo_local_comments')),
+			],
+		];
+
+		$statistics = $this->getJobsPerPriority($statistics);
+
+		$this->jsonExit($statistics);
+	}
+
+	private function isAllowed(array $request): bool
+	{
+		return empty(!$request['key']) && $request['key'] == $this->config->get('system', 'stats_key');
+	}
+
+	private function getJobsPerPriority(array $statistics): array
+	{
+		$statistics['worker']['active'] = $statistics['worker']['total'] = [
+			Worker::PRIORITY_UNDEFINED  => 0,
+			Worker::PRIORITY_CRITICAL   => 0,
+			Worker::PRIORITY_HIGH       => 0,
+			Worker::PRIORITY_MEDIUM     => 0,
+			Worker::PRIORITY_LOW        => 0,
+			Worker::PRIORITY_NEGLIGIBLE => 0,
+			'total'                     => 0,
+		];
+
+		for ($i = 1; $i <= $this->config->get('system', 'worker_defer_limit'); $i++) {
+			$statistics['worker']['deferred'][$i] = 0;
+		}
+		$statistics['worker']['deferred']['total'] = 0;
+
+		$jobs = $this->dba->p("SELECT COUNT(*) AS `entries`, `priority` FROM `workerqueue` WHERE NOT `done` AND `retrial` = ? GROUP BY `priority`", 0);
+		while ($entry = $this->dba->fetch($jobs)) {
+			$running  = $this->dba->count('workerqueue-view', ['priority' => $entry['priority']]);
+			$statistics['worker']['active']['total'] += $running;
+			$statistics['worker']['active'][$entry['priority']] = $running;
+			$statistics['worker']['total']['total'] += $entry['entries'];
+			$statistics['worker']['total'][$entry['priority']] = $entry['entries'];
+		}
+		$this->dba->close($jobs);
+		$statistics['worker']['active'][Worker::PRIORITY_UNDEFINED] = max(0, Worker::activeWorkers() - $statistics['worker']['active']['total']);
+
+		$jobs = $this->dba->p("SELECT COUNT(*) AS `entries`, `retrial` FROM `workerqueue` WHERE NOT `done` AND `retrial` > ? GROUP BY `retrial`", 0);
+		while ($entry = $this->dba->fetch($jobs)) {
+			$statistics['worker']['deferred']['total'] += $entry['entries'];
+			$statistics['worker']['deferred'][$entry['retrial']] = $entry['entries'];
+		}
+		$this->dba->close($jobs);
+
+		return $statistics;
+	}
+}

static/defaults.config.php

@@ -556,6 +556,10 @@ return [
 		// Show all themes including the unsupported ones.
 		'show_unsupported_themes' => false,
 
+		// stats_key (String)
+		// A random string to be added to the /stats?key=... endpoint to enable the monitoring statistics
+		'stats_key' => '',
+
 		// throttle_limit_day (Integer)
 		// Maximum number of posts that a user can send per day with the API. 0 to disable daily throttling.
 		'throttle_limit_day' => 0,

static/routes.config.php

@@ -658,6 +658,8 @@ return [
 		],
 	],
 
+	'/stats' => [Module\Stats::class, [R::GET]],
+
 	'/network' => [
 		'[/{content}]'                => [Module\Conversation\Network::class, [R::GET]],
 		'/archive/{from:\d\d\d\d-\d\d-\d\d}[/{to:\d\d\d\d-\d\d-\d\d}]' => [Module\Conversation\Network::class, [R::GET]],