From e319621f6368a226b12ea825bbbad2a5c516b8a4 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 23 Jun 2024 14:38:47 +0000 Subject: [PATCH] Nodeinfo: We now parse Nodeinfo 2.1 and 2.2 as well --- database.sql | 8 +-- doc/database/db_gserver.md | 68 +++++++++++++------------ src/Model/GServer.php | 95 +++++++++++++++++++++++++---------- static/dbstructure.config.php | 8 +-- 4 files changed, 113 insertions(+), 66 deletions(-) diff --git a/database.sql b/database.sql index fd9eb1c28f..148dcd2626 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2024.06-rc (Yellow Archangel) --- DB_UPDATE_VERSION 1567 +-- DB_UPDATE_VERSION 1568 -- ------------------------------------------ @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS `gserver` ( `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', `url` varbinary(383) NOT NULL DEFAULT '' COMMENT '', `nurl` varbinary(383) NOT NULL DEFAULT '' COMMENT '', - `version` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `version` varchar(255) NOT NULL DEFAULT '' COMMENT 'The version of this server software.', `site_name` varchar(255) NOT NULL DEFAULT '' COMMENT '', `info` text COMMENT '', `register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '', @@ -28,7 +28,9 @@ CREATE TABLE IF NOT EXISTS `gserver` ( `noscrape` varbinary(383) NOT NULL DEFAULT '' COMMENT '', `network` char(4) NOT NULL DEFAULT '' COMMENT '', `protocol` tinyint unsigned COMMENT 'The protocol of the server', - `platform` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `platform` varchar(255) NOT NULL DEFAULT '' COMMENT 'The canonical name of this server software.', + `repository` varbinary(383) COMMENT 'The url of the source code repository of this server software.', + `homepage` varbinary(383) COMMENT 'The url of the homepage of this server software.', `relay-subscribe` boolean NOT NULL DEFAULT '0' COMMENT 'Has the server subscribed to the relay system', `relay-scope` varchar(10) NOT NULL DEFAULT '' COMMENT 'The scope of messages that the server wants to get', `detection-method` tinyint unsigned COMMENT 'Method that had been used to detect that server', diff --git a/doc/database/db_gserver.md b/doc/database/db_gserver.md index 238f053d5d..0d64ef27a1 100644 --- a/doc/database/db_gserver.md +++ b/doc/database/db_gserver.md @@ -6,39 +6,41 @@ Global servers Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| --------------------- | -------------------------------------------------- | ---------------- | ---- | --- | ------------------- | -------------- | -| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | -| url | | varbinary(383) | NO | | | | -| nurl | | varbinary(383) | NO | | | | -| version | | varchar(255) | NO | | | | -| site_name | | varchar(255) | NO | | | | -| info | | text | YES | | NULL | | -| register_policy | | tinyint | NO | | 0 | | -| registered-users | Number of registered users | int unsigned | NO | | 0 | | -| active-week-users | Number of active users in the last week | int unsigned | YES | | NULL | | -| active-month-users | Number of active users in the last month | int unsigned | YES | | NULL | | -| active-halfyear-users | Number of active users in the last six month | int unsigned | YES | | NULL | | -| local-posts | Number of local posts | int unsigned | YES | | NULL | | -| local-comments | Number of local comments | int unsigned | YES | | NULL | | -| directory-type | Type of directory service (Poco, Mastodon) | tinyint | YES | | 0 | | -| poco | | varbinary(383) | NO | | | | -| openwebauth | Path to the OpenWebAuth endpoint | varbinary(383) | YES | | NULL | | -| authredirect | Path to the authRedirect endpoint | varbinary(383) | YES | | NULL | | -| noscrape | | varbinary(383) | NO | | | | -| network | | char(4) | NO | | | | -| protocol | The protocol of the server | tinyint unsigned | YES | | NULL | | -| platform | | varchar(255) | NO | | | | -| relay-subscribe | Has the server subscribed to the relay system | boolean | NO | | 0 | | -| relay-scope | The scope of messages that the server wants to get | varchar(10) | NO | | | | -| detection-method | Method that had been used to detect that server | tinyint unsigned | YES | | NULL | | -| created | | datetime | NO | | 0001-01-01 00:00:00 | | -| last_poco_query | | datetime | YES | | 0001-01-01 00:00:00 | | -| last_contact | Last successful connection request | datetime | YES | | 0001-01-01 00:00:00 | | -| last_failure | Last failed connection request | datetime | YES | | 0001-01-01 00:00:00 | | -| blocked | Server is blocked | boolean | YES | | NULL | | -| failed | Connection failed | boolean | YES | | NULL | | -| next_contact | Next connection request | datetime | YES | | 0001-01-01 00:00:00 | | +| Field | Description | Type | Null | Key | Default | Extra | +| --------------------- | -------------------------------------------------------------- | ---------------- | ---- | --- | ------------------- | -------------- | +| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | +| url | | varbinary(383) | NO | | | | +| nurl | | varbinary(383) | NO | | | | +| version | The version of this server software. | varchar(255) | NO | | | | +| site_name | | varchar(255) | NO | | | | +| info | | text | YES | | NULL | | +| register_policy | | tinyint | NO | | 0 | | +| registered-users | Number of registered users | int unsigned | NO | | 0 | | +| active-week-users | Number of active users in the last week | int unsigned | YES | | NULL | | +| active-month-users | Number of active users in the last month | int unsigned | YES | | NULL | | +| active-halfyear-users | Number of active users in the last six month | int unsigned | YES | | NULL | | +| local-posts | Number of local posts | int unsigned | YES | | NULL | | +| local-comments | Number of local comments | int unsigned | YES | | NULL | | +| directory-type | Type of directory service (Poco, Mastodon) | tinyint | YES | | 0 | | +| poco | | varbinary(383) | NO | | | | +| openwebauth | Path to the OpenWebAuth endpoint | varbinary(383) | YES | | NULL | | +| authredirect | Path to the authRedirect endpoint | varbinary(383) | YES | | NULL | | +| noscrape | | varbinary(383) | NO | | | | +| network | | char(4) | NO | | | | +| protocol | The protocol of the server | tinyint unsigned | YES | | NULL | | +| platform | The canonical name of this server software. | varchar(255) | NO | | | | +| repository | The url of the source code repository of this server software. | varbinary(383) | YES | | NULL | | +| homepage | The url of the homepage of this server software. | varbinary(383) | YES | | NULL | | +| relay-subscribe | Has the server subscribed to the relay system | boolean | NO | | 0 | | +| relay-scope | The scope of messages that the server wants to get | varchar(10) | NO | | | | +| detection-method | Method that had been used to detect that server | tinyint unsigned | YES | | NULL | | +| created | | datetime | NO | | 0001-01-01 00:00:00 | | +| last_poco_query | | datetime | YES | | 0001-01-01 00:00:00 | | +| last_contact | Last successful connection request | datetime | YES | | 0001-01-01 00:00:00 | | +| last_failure | Last failed connection request | datetime | YES | | 0001-01-01 00:00:00 | | +| blocked | Server is blocked | boolean | YES | | NULL | | +| failed | Connection failed | boolean | YES | | NULL | | +| next_contact | Next connection request | datetime | YES | | 0001-01-01 00:00:00 | | Indexes ------------ diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 6129f84d3c..6b3ef6e220 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -87,9 +87,11 @@ class GServer // Standardized endpoints const DETECT_STATISTICS_JSON = 100; - const DETECT_NODEINFO_1 = 101; - const DETECT_NODEINFO_2 = 102; - const DETECT_NODEINFO_210 = 103; + const DETECT_NODEINFO_10 = 101; // Nodeinfo Version 1.0 + const DETECT_NODEINFO_20 = 102; // Nodeinfo Version 2.0 + const DETECT_NODEINFO2_10 = 103; // Nodeinfo2 Version 1.0 + const DETECT_NODEINFO_21 = 104; // Nodeinfo Version 2.1 + const DETECT_NODEINFO_22 = 105; // Nodeinfo Version 2.2 /** * Check for the existence of a server and adds it in the background if not existant @@ -612,7 +614,7 @@ class GServer $in_webroot = empty(parse_url($url, PHP_URL_PATH)); // When a nodeinfo is present, we don't need to dig further - $curlResult = DI::httpClient()->get($url . '/.well-known/x-nodeinfo2', HttpClientAccept::JSON, [HttpClientOptions::REQUEST => HttpClientRequest::SERVERINFO]); + $curlResult = DI::httpClient()->get($url . '/.well-known/nodeinfo', HttpClientAccept::JSON, [HttpClientOptions::REQUEST => HttpClientRequest::SERVERINFO]); if ($curlResult->isTimeout()) { self::setFailureByUrl($url); return false; @@ -621,10 +623,11 @@ class GServer if (!empty($network) && !in_array($network, Protocol::NATIVE_SUPPORT)) { $serverdata = ['detection-method' => self::DETECT_MANUAL, 'network' => $network, 'platform' => '', 'version' => '', 'site_name' => '', 'info' => '']; } else { - $serverdata = self::parseNodeinfo210($curlResult); - if (empty($serverdata)) { - $curlResult = DI::httpClient()->get($url . '/.well-known/nodeinfo', HttpClientAccept::JSON, [HttpClientOptions::REQUEST => HttpClientRequest::SERVERINFO]); - $serverdata = self::fetchNodeinfo($url, $curlResult); + $serverdata = self::parseNodeinfo($url, $curlResult); + + if (empty($serverdata) || !in_array($serverdata['detection-method'], [self::DETECT_NODEINFO_20, self::DETECT_NODEINFO_21, self::DETECT_NODEINFO_22])) { + $curlResult = DI::httpClient()->get($url . '/.well-known/x-nodeinfo2', HttpClientAccept::JSON, [HttpClientOptions::REQUEST => HttpClientRequest::SERVERINFO]); + $serverdata = self::parseNodeinfo2($curlResult) ?: $serverdata; } } @@ -1049,7 +1052,9 @@ class GServer } /** - * Detect server type by using the nodeinfo data + * Parses Nodeinfo + * + * @see https://github.com/jhass/nodeinfo * * @param string $url address of the server * @param ICanHandleHttpResponses $httpResult @@ -1058,7 +1063,7 @@ class GServer * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function fetchNodeinfo(string $url, ICanHandleHttpResponses $httpResult): array + private static function parseNodeinfo(string $url, ICanHandleHttpResponses $httpResult): array { if (!$httpResult->isSuccess()) { return []; @@ -1072,6 +1077,7 @@ class GServer $nodeinfo1_url = ''; $nodeinfo2_url = ''; + $detection_method = self::DETECT_MANUAL; foreach ($nodeinfo['links'] as $link) { if (!is_array($link) || empty($link['rel']) || empty($link['href'])) { @@ -1081,8 +1087,15 @@ class GServer if ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/1.0') { $nodeinfo1_url = Network::addBasePath($link['href'], $httpResult->getUrl()); - } elseif ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.0') { + } elseif (($detection_method < self::DETECT_NODEINFO_20) && ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.0')) { $nodeinfo2_url = Network::addBasePath($link['href'], $httpResult->getUrl()); + $detection_method = self::DETECT_NODEINFO_20; + } elseif (($detection_method < self::DETECT_NODEINFO_21) && ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.1')) { + $nodeinfo2_url = Network::addBasePath($link['href'], $httpResult->getUrl()); + $detection_method = self::DETECT_NODEINFO_21; + } elseif (($detection_method < self::DETECT_NODEINFO_22) && ($link['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.2')) { + $nodeinfo2_url = Network::addBasePath($link['href'], $httpResult->getUrl()); + $detection_method = self::DETECT_NODEINFO_22; } } @@ -1093,18 +1106,20 @@ class GServer $server = []; if (!empty($nodeinfo2_url)) { - $server = self::parseNodeinfo2($nodeinfo2_url); + $server = self::parseNodeinfo_2($nodeinfo2_url, $detection_method); } if (empty($server) && !empty($nodeinfo1_url)) { - $server = self::parseNodeinfo1($nodeinfo1_url); + $server = self::parseNodeinfo_1($nodeinfo1_url); } return $server; } /** - * Parses Nodeinfo 1 + * Parses Nodeinfo with the version 1.0 + * + * @see https://github.com/jhass/nodeinfo/tree/main/schemas/1.0 * * @param string $nodeinfo_url address of the nodeinfo path * @@ -1112,7 +1127,7 @@ class GServer * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo1(string $nodeinfo_url): array + private static function parseNodeinfo_1(string $nodeinfo_url): array { $curlResult = DI::httpClient()->get($nodeinfo_url, HttpClientAccept::JSON, [HttpClientOptions::REQUEST => HttpClientRequest::SERVERINFO]); if (!$curlResult->isSuccess()) { @@ -1125,8 +1140,10 @@ class GServer return []; } - $server = ['detection-method' => self::DETECT_NODEINFO_1, - 'register_policy' => Register::CLOSED]; + $server = [ + 'detection-method' => self::DETECT_NODEINFO_10, + 'register_policy' => Register::CLOSED + ]; if (!empty($nodeinfo['openRegistrations'])) { $server['register_policy'] = Register::OPEN; @@ -1202,17 +1219,20 @@ class GServer } /** - * Parses Nodeinfo 2 + * Parses Nodeinfo with the versions 2.0, 2.1 and 2.2 * - * @see https://git.feneas.org/jaywink/nodeinfo2 + * @see https://github.com/jhass/nodeinfo/tree/main/schemas/2.0 + * @see https://github.com/jhass/nodeinfo/tree/main/schemas/2.1 + * @see https://github.com/jhass/nodeinfo/tree/main/schemas/2.2 * - * @param string $nodeinfo_url address of the nodeinfo path + * @param string $nodeinfo_url address of the nodeinfo path + * @param int $detection_method nodeinfo version * * @return array Server data * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo2(string $nodeinfo_url): array + private static function parseNodeinfo_2(string $nodeinfo_url, int $detection_method): array { $curlResult = DI::httpClient()->get($nodeinfo_url, HttpClientAccept::JSON, [HttpClientOptions::REQUEST => HttpClientRequest::SERVERINFO]); if (!$curlResult->isSuccess()) { @@ -1225,7 +1245,7 @@ class GServer } $server = [ - 'detection-method' => self::DETECT_NODEINFO_2, + 'detection-method' => $detection_method, 'register_policy' => Register::CLOSED, 'platform' => 'unknown', ]; @@ -1234,6 +1254,15 @@ class GServer $server['register_policy'] = Register::OPEN; } + if (!empty($nodeinfo['instance'])) { + if (!empty($nodeinfo['instance']['name'])) { + $server['site_name'] = $nodeinfo['instance']['name']; + } + if (!empty($nodeinfo['instance']['description'])) { + $server['info'] = $nodeinfo['instance']['description']; + } + } + if (!empty($nodeinfo['software'])) { if (isset($nodeinfo['software']['name'])) { $server['platform'] = strtolower($nodeinfo['software']['name']); @@ -1249,6 +1278,13 @@ class GServer if (($server['platform'] == 'mastodon') && substr($nodeinfo['software']['version'], -5) == '-qoto') { $server['platform'] = 'qoto'; } + + if (isset($nodeinfo['software']['repository'])) { + $server['repository'] = strtolower($nodeinfo['software']['repository']); + } + if (isset($nodeinfo['software']['homepage'])) { + $server['homepage'] = strtolower($nodeinfo['software']['homepage']); + } } } @@ -1260,6 +1296,9 @@ class GServer if (!empty($nodeinfo['metadata']['nodeName'])) { $server['site_name'] = $nodeinfo['metadata']['nodeName']; } + if (!empty($nodeinfo['metadata']['nodeDescription'])) { + $server['info'] = $nodeinfo['metadata']['nodeDescription']; + } if (!empty($nodeinfo['usage']['users']['total'])) { $server['registered-users'] = max($nodeinfo['usage']['users']['total'], 1); @@ -1320,9 +1359,9 @@ class GServer } /** - * Parses NodeInfo2 protocol 1.0 + * Parses NodeInfo2 * - * @see https://github.com/jaywink/nodeinfo2/blob/master/PROTOCOL.md + * @see https://github.com/jaywink/nodeinfo2 * * @param string $nodeinfo_url address of the nodeinfo path * @@ -1330,7 +1369,7 @@ class GServer * * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function parseNodeinfo210(ICanHandleHttpResponses $httpResult): array + private static function parseNodeinfo2(ICanHandleHttpResponses $httpResult): array { if (!$httpResult->isSuccess()) { return []; @@ -1342,8 +1381,10 @@ class GServer return []; } - $server = ['detection-method' => self::DETECT_NODEINFO_210, - 'register_policy' => Register::CLOSED]; + $server = [ + 'detection-method' => self::DETECT_NODEINFO2_10, + 'register_policy' => Register::CLOSED + ]; if (!empty($nodeinfo['openRegistrations'])) { $server['register_policy'] = Register::OPEN; diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 77e337eb65..2451270c53 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -56,7 +56,7 @@ use Friendica\Database\DBA; // This file is required several times during the test in DbaDefinition which justifies this condition if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1567); + define('DB_UPDATE_VERSION', 1568); } return [ @@ -67,7 +67,7 @@ return [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], "url" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""], "nurl" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""], - "version" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "version" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "The version of this server software."], "site_name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "info" => ["type" => "text", "comment" => ""], "register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""], @@ -84,7 +84,9 @@ return [ "noscrape" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], "protocol" => ["type" => "tinyint unsigned", "comment" => "The protocol of the server"], - "platform" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "platform" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "The canonical name of this server software."], + "repository" => ["type" => "varbinary(383)", "comment" => "The url of the source code repository of this server software."], + "homepage" => ["type" => "varbinary(383)", "comment" => "The url of the homepage of this server software."], "relay-subscribe" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Has the server subscribed to the relay system"], "relay-scope" => ["type" => "varchar(10)", "not null" => "1", "default" => "", "comment" => "The scope of messages that the server wants to get"], "detection-method" => ["type" => "tinyint unsigned", "comment" => "Method that had been used to detect that server"],