Merge pull request #8059 from annando/item-notification

Item notification data is now stored in the "user-item" table

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 2019.12-rc (Dalmatian Bellflower)
+-- Friendica 2020.03-dev (Dalmatian Bellflower)
--- DB_UPDATE_VERSION 1328
+-- DB_UPDATE_VERSION 1329
 -- ------------------------------------------
 
 
@@ -1285,8 +1285,10 @@ CREATE TABLE IF NOT EXISTS `user-item` (
 	`hidden` boolean NOT NULL DEFAULT '0' COMMENT 'Marker to hide an item from the user',
 	`ignored` boolean COMMENT 'Ignore this thread if set',
 	`pinned` boolean COMMENT 'The item is pinned on the profile page',
+	`notification-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
 	 PRIMARY KEY(`uid`,`iid`),
-	 INDEX `uid_pinned` (`uid`,`pinned`)
+	 INDEX `uid_pinned` (`uid`,`pinned`),
+	 INDEX `iid_uid` (`iid`,`uid`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='User specific item data';
 
 --

include/api.php

@@ -29,6 +29,7 @@ use Friendica\Model\Mail;
 use Friendica\Model\Photo;
 use Friendica\Model\Profile;
 use Friendica\Model\User;
+use Friendica\Model\UserItem;
 use Friendica\Network\FKOAuth1;
 use Friendica\Network\HTTPException;
 use Friendica\Network\HTTPException\BadRequestException;
@@ -1519,7 +1520,7 @@ function api_search($type)
 	} elseif (!empty($_REQUEST['count'])) {
 		$count = $_REQUEST['count'];
 	}
-	
+
 	$since_id = $_REQUEST['since_id'] ?? 0;
 	$max_id = $_REQUEST['max_id'] ?? 0;
 	$page = $_REQUEST['page'] ?? 1;
@@ -1555,7 +1556,7 @@ function api_search($type)
 
 		$condition = [implode(' AND ', $preCondition)];
 	} else {
-		$condition = ["`id` > ? 
+		$condition = ["`id` > ?
 			" . ($exclude_replies ? " AND `id` = `parent` " : ' ') . "
 			AND (`uid` = 0 OR (`uid` = ? AND NOT `global`))
 			AND `body` LIKE CONCAT('%',?,'%')",
@@ -2158,17 +2159,34 @@ function api_statuses_mentions($type)
 
 	$start = max(0, ($page - 1) * $count);
 
-	$condition = ["`uid` = ? AND `gravity` IN (?, ?) AND `item`.`id` > ? AND `author-id` != ? AND `mention`
+	$query = "SELECT `item`.`id` FROM `user-item`
-		AND `item`.`parent` IN (SELECT `iid` FROM `thread` WHERE `thread`.`uid` = ? AND NOT `thread`.`ignored`)",
+		INNER JOIN `item` ON `item`.`id` = `user-item`.`iid` AND `item`.`gravity` IN (?, ?)
-		api_user(), GRAVITY_PARENT, GRAVITY_COMMENT, $since_id, $user_info['pid'], api_user()];
+		WHERE (`user-item`.`hidden` IS NULL OR NOT `user-item`.`hidden`) AND
+			`user-item`.`uid` = ? AND `user-item`.`notification-type` & ? != 0
+			AND `user-item`.`iid` > ?";
+	$condition = [GRAVITY_PARENT, GRAVITY_COMMENT, api_user(),
+		UserItem::NOTIF_EXPLICIT_TAGGED | UserItem::NOTIF_IMPLICIT_TAGGED |
+		UserItem::NOTIF_THREAD_COMMENT | UserItem::NOTIF_DIRECT_COMMENT,
+		$since_id];
 
 	if ($max_id > 0) {
-		$condition[0] .= " AND `item`.`id` <= ?";
+		$query .= " AND `item`.`id` <= ?";
 		$condition[] = $max_id;
 	}
 
+	$query .= " ORDER BY `user-item`.`iid` DESC LIMIT ?, ?";
+	$condition[] = $start;
+	$condition[] = $count;
+
+	$useritems = DBA::p($query, $condition);
+	$itemids = [];
+	while ($useritem = DBA::fetch($useritems)) {
+		$itemids[] = $useritem['id'];
+	}
+	DBA::close($useritems);
+
 	$params = ['order' => ['id' => true], 'limit' => [$start, $count]];
-	$statuses = Item::selectForUser(api_user(), [], $condition, $params);
+	$statuses = Item::selectForUser(api_user(), [], ['id' => $itemids], $params);
 
 	$ret = api_format_items(Item::inArray($statuses), $user_info, false, $type);
 

src/Database/PostUpdate.php

@@ -10,6 +10,7 @@ use Friendica\Core\Protocol;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
 use Friendica\Model\ItemURI;
+use Friendica\Model\UserItem;
 use Friendica\Model\PermissionSet;
 
 /**
@@ -40,6 +41,9 @@ class PostUpdate
 		if (!self::update1322()) {
 			return false;
 		}
+		if (!self::update1329()) {
+			return false;
+		}
 
 		return true;
 	}
@@ -453,4 +457,54 @@ class PostUpdate
 
 		return true;
 	}
+
+	/**
+	 * @brief update user-item data with notifications
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 */
+	private static function update1329()
+	{
+		// Was the script completed?
+		if (Config::get('system', 'post_update_version') >= 1329) {
+			return true;
+		}
+
+		$id = Config::get('system', 'post_update_version_1329_id', 0);
+
+		Logger::info('Start', ['item' => $id]);
+
+		$start_id = $id;
+		$rows = 0;
+		$condition = ["`id` > ?", $id];
+		$params = ['order' => ['id'], 'limit' => 10000];
+		$items = DBA::select('item', ['id'], $condition, $params);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($item = DBA::fetch($items)) {
+			$id = $item['id'];
+
+			UserItem::setNotification($item['id']);
+
+			++$rows;
+		}
+		DBA::close($items);
+
+		Config::set('system', 'post_update_version_1329_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($start_id == $id) {
+			Config::set('system', 'post_update_version', 1329);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
 }

src/Model/Item.php

@@ -656,7 +656,7 @@ class Item
 			'iaid' => 'internal-iaid'];
 
 		if ($usermode) {
-			$fields['user-item'] = ['pinned', 'ignored' => 'internal-user-ignored'];
+			$fields['user-item'] = ['pinned', 'notification-type', 'ignored' => 'internal-user-ignored'];
 		}
 
 		$fields['item-activity'] = ['activity', 'activity' => 'internal-activity'];
@@ -2026,6 +2026,8 @@ class Item
 
 		self::updateContact($item);
 
+		UserItem::setNotification($current_post);
+
 		check_user_notification($current_post);
 
 		if ($notify || ($item['visible'] && ((!empty($parent) && $parent['origin']) || $item['origin']))) {

src/Model/UserItem.php

@@ -0,0 +1,285 @@
+<?php
+
+/**
+ * @file src/Model/UserItem.php
+ */
+
+namespace Friendica\Model;
+
+use Friendica\Core\Logger;
+use Friendica\Core\Hook;
+use Friendica\Database\DBA;
+use Friendica\DI;
+use Friendica\Util\Strings;
+
+class UserItem
+{
+	// Notification types
+	const NOTIF_NONE = 0;
+	const NOTIF_EXPLICIT_TAGGED = 1;
+	const NOTIF_IMPLICIT_TAGGED = 2;
+	const NOTIF_THREAD_COMMENT = 4;
+	const NOTIF_DIRECT_COMMENT = 8;
+	const NOTIF_COMMENT_PARTICIPATION = 16;
+	const NOTIF_ACTIVITY_PARTICIPATION = 32;
+	const NOTIF_SHARED = 128;
+
+	/**
+	 * Checks an item for notifications and sets the "notification-type" field
+	 * @ToDo:
+	 * - Check for mentions in posts with "uid=0" where the user hadn't interacted before
+	 *
+	 * @param int $iid Item ID
+	 */
+	public static function setNotification(int $iid)
+	{
+		$fields = ['id', 'uid', 'body', 'parent', 'gravity', 'tag', 'contact-id', 'thr-parent', 'parent-uri', 'author-id'];
+		$item = Item::selectFirst($fields, ['id' => $iid, 'origin' => false]);
+		if (!DBA::isResult($item)) {
+			return;
+		}
+
+		// fetch all users in the thread
+		$users = DBA::p("SELECT DISTINCT(`contact`.`uid`) FROM `item`
+			INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND `contact`.`uid` != 0
+			WHERE `parent` IN (SELECT `parent` FROM `item` WHERE `id`=?)", $iid);
+		while ($user = DBA::fetch($users)) {
+			self::setNotificationForUser($item, $user['uid']);
+		}
+		DBA::close($users);
+	}
+
+	/**
+	 * Checks an item for notifications for the given user and sets the "notification-type" field
+	 *
+	 * @param array $item Item array
+	 * @param int   $uid  User ID
+	 */
+	private static function setNotificationForUser(array $item, int $uid)
+	{
+		$thread = Item::selectFirstThreadForUser($uid, ['ignored'], ['iid' => $item['parent'], 'deleted' => false]);
+		if ($thread['ignored']) {
+			return;
+		}
+
+		$notification_type = self::NOTIF_NONE;
+
+		if (self::checkShared($item, $uid)) {
+			$notification_type = $notification_type | self::NOTIF_SHARED;
+		}
+
+		$profiles = self::getProfileForUser($uid);
+
+		// Fetch all contacts for the given profiles
+		$contacts = [];
+		$ret = DBA::select('contact', ['id'], ['uid' => 0, 'nurl' => $profiles]);
+		while ($contact = DBA::fetch($ret)) {
+			$contacts[] = $contact['id'];
+		}
+		DBA::close($ret);
+
+		// Don't create notifications for user's posts
+		if (in_array($item['author-id'], $contacts)) {
+			return;
+		}
+
+		if (self::checkImplicitMention($item, $profiles)) {
+			$notification_type = $notification_type | self::NOTIF_IMPLICIT_TAGGED;
+		}
+
+		if (self::checkExplicitMention($item, $profiles)) {
+			$notification_type = $notification_type | self::NOTIF_EXPLICIT_TAGGED;
+		}
+
+		if (self::checkCommentedThread($item, $contacts)) {
+			$notification_type = $notification_type | self::NOTIF_THREAD_COMMENT;
+		}
+
+		if (self::checkDirectComment($item, $contacts)) {
+			$notification_type = $notification_type | self::NOTIF_DIRECT_COMMENT;
+		}
+
+		if (self::checkCommentedParticipation($item, $contacts)) {
+			$notification_type = $notification_type | self::NOTIF_COMMENT_PARTICIPATION;
+		}
+
+		if (self::checkActivityParticipation($item, $contacts)) {
+			$notification_type = $notification_type | self::NOTIF_ACTIVITY_PARTICIPATION;
+		}
+
+		if (empty($notification_type)) {
+			return;
+		}
+
+		Logger::info('Set notification', ['iid' => $item['id'], 'uid' => $uid, 'notification-type' => $notification_type]);
+
+		DBA::update('user-item', ['notification-type' => $notification_type], ['iid' => $item['id'], 'uid' => $uid], true);
+	}
+
+	/**
+	 * Fetch all profiles (contact URL) of a given user
+	 * @param int $uid User ID
+	 *
+	 * @return array Profile links
+	 */
+	private static function getProfileForUser(int $uid)
+	{
+		$notification_data = ['uid' => $uid, 'profiles' => []];
+		Hook::callAll('check_item_notification', $notification_data);
+
+		$profiles = $notification_data['profiles'];
+
+		$user = DBA::selectFirst('user', ['nickname'], ['uid' => $uid]);
+		if (!DBA::isResult($user)) {
+			return [];
+		}
+
+		$owner = DBA::selectFirst('contact', ['url', 'alias'], ['self' => true, 'uid' => $uid]);
+		if (!DBA::isResult($owner)) {
+			return [];
+		}
+
+		// This is our regular URL format
+		$profiles[] = $owner['url'];
+
+		// Now the alias
+		$profiles[] = $owner['alias'];
+
+		// Notifications from Diaspora are often with an URL in the Diaspora format
+		$profiles[] = DI::baseUrl() . '/u/' . $user['nickname'];
+
+		// Validate and add profile links
+		foreach ($profiles AS $key => $profile) {
+			// Check for invalid profile urls (without scheme, host or path) and remove them
+			if (empty(parse_url($profile, PHP_URL_SCHEME)) || empty(parse_url($profile, PHP_URL_HOST)) || empty(parse_url($profile, PHP_URL_PATH))) {
+				unset($profiles[$key]);
+				continue;
+			}
+
+			// Add the normalized form
+			$profile = Strings::normaliseLink($profile);
+			$profiles[] = $profile;
+
+			// Add the SSL form
+			$profile = str_replace('http://', 'https://', $profile);
+			$profiles[] = $profile;
+		}
+
+		return array_unique($profiles);
+	}
+
+	/**
+	 * Check for a "shared" notification for every new post of contacts from the given user
+	 * @param array $item
+	 * @param int   $uid  User ID
+	 * @return bool A contact had shared something
+	 */
+	private static function checkShared(array $item, int $uid)
+	{
+		if ($item['gravity'] != GRAVITY_PARENT) {
+			return false;
+		}
+
+		// Either the contact had posted something directly
+		if (DBA::exists('contact', ['id' => $item['contact-id'], 'notify_new_posts' => true])) {
+			return true;
+		}
+
+		// Or the contact is a mentioned forum
+		$tags = DBA::select('term', ['url'], ['otype' => TERM_OBJ_POST, 'oid' => $item['id'], 'type' => TERM_MENTION, 'uid' => $uid]);
+		while ($tag = DBA::fetch($tags)) {
+			$condition = ['nurl' => Strings::normaliseLink($tag['url']), 'uid' => $uid, 'notify_new_posts' => true, 'contact-type' => Contact::TYPE_COMMUNITY];
+			if (DBA::exists('contact', $condition)) {
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Check for an implicit mention (only tag, no body) of the given user
+	 * @param array $item
+	 * @param array $profiles Profile links
+	 * @return bool The user is mentioned
+	 */
+	private static function checkImplicitMention(array $item, array $profiles)
+	{
+		foreach ($profiles AS $profile) {
+			if (strpos($item['tag'], '=' . $profile.']') || strpos($item['body'], '=' . $profile . ']')) {
+				if (strpos($item['body'], $profile) === false) {
+					return true;
+				}
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Check for an explicit mention (tag and body) of the given user
+	 * @param array $item
+	 * @param array $profiles Profile links
+	 * @return bool The user is mentioned
+	 */
+	private static function checkExplicitMention(array $item, array $profiles)
+	{
+		foreach ($profiles AS $profile) {
+			if (strpos($item['tag'], '=' . $profile.']') || strpos($item['body'], '=' . $profile . ']')) {
+				if (!(strpos($item['body'], $profile) === false)) {
+					return true;
+				}
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Check if the given user had created this thread
+	 * @param array $item
+	 * @param array $contacts Array of contact IDs
+	 * @return bool The user had created this thread
+	 */
+	private static function checkCommentedThread(array $item, array $contacts)
+	{
+		$condition = ['parent' => $item['parent'], 'author-id' => $contacts, 'deleted' => false, 'gravity' => GRAVITY_PARENT];
+		return Item::exists($condition);
+	}
+
+	/**
+	 * Check for a direct comment to a post of the given user
+	 * @param array $item
+	 * @param array $contacts Array of contact IDs
+	 * @return bool The item is a direct comment to a user comment
+	 */
+	private static function checkDirectComment(array $item, array $contacts)
+	{
+		$condition = ['uri' => $item['thr-parent'], 'uid' => $item['uid'], 'author-id' => $contacts, 'deleted' => false, 'gravity' => GRAVITY_COMMENT];
+		return Item::exists($condition);
+	}
+
+	/**
+	 *  Check if the user had commented in this thread
+	 * @param array $item
+	 * @param array $contacts Array of contact IDs
+	 * @return bool The user had commented in the thread
+	 */
+	private static function checkCommentedParticipation(array $item, array $contacts)
+	{
+		$condition = ['parent' => $item['parent'], 'author-id' => $contacts, 'deleted' => false, 'gravity' => GRAVITY_COMMENT];
+		return Item::exists($condition);
+	}
+
+	/**
+	 * Check if the user had interacted in this thread (Like, Dislike, ...)
+	 * @param array $item
+	 * @param array $contacts Array of contact IDs
+	 * @return bool The user had interacted in the thread
+	 */
+	private static function checkActivityParticipation(array $item, array $contacts)
+	{
+		$condition = ['parent' => $item['parent'], 'author-id' => $contacts, 'deleted' => false, 'gravity' => GRAVITY_ACTIVITY];
+		return Item::exists($condition);
+	}
+}

static/dbstructure.config.php

@@ -34,7 +34,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1328);
+	define('DB_UPDATE_VERSION', 1329);
 }
 
 return [
@@ -1388,11 +1388,13 @@ return [
 			"uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["user" => "uid"], "comment" => "User id"],
 			"hidden" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Marker to hide an item from the user"],
 			"ignored" => ["type" => "boolean", "comment" => "Ignore this thread if set"],
-			"pinned" => ["type" => "boolean", "comment" => "The item is pinned on the profile page"]
+			"pinned" => ["type" => "boolean", "comment" => "The item is pinned on the profile page"],
+			"notification-type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
 		],
 		"indexes" => [
 			"PRIMARY" => ["uid", "iid"],
-			"uid_pinned" => ["uid", "pinned"]
+			"uid_pinned" => ["uid", "pinned"],
+			"iid_uid" => ["iid", "uid"]
 		]
 	],
 	"worker-ipc" => [