From e5c7ce090299102f427bdf427f5347ef6c17d121 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 14 Oct 2016 05:45:32 +0000 Subject: [PATCH] Use "last-item" in the contact table instead of an expensive sql query --- include/dba.php | 36 ++++++++++++++++++ include/dbm.php | 4 ++ include/event.php | 4 +- include/items.php | 83 +++++++++++++++++++++++------------------ include/post_update.php | 44 +++++++++++++++++++++- mod/admin.php | 17 ++++----- mod/item.php | 2 + mod/nodeinfo.php | 14 +++---- 8 files changed, 148 insertions(+), 56 deletions(-) diff --git a/include/dba.php b/include/dba.php index 86a3762b44..2d96886dce 100644 --- a/include/dba.php +++ b/include/dba.php @@ -343,6 +343,42 @@ function q($sql) { }} +/** + * @brief Performs a query with "dirty reads" + * + * By doing dirty reads (reading uncommitted data) no locks are performed + * This function can be used to fetch data that doesn't need to be reliable. + * + * @param $args Query parameters (1 to N parameters of different types) + * @return array Query array + */ +function qu($sql) { + + global $db; + $args = func_get_args(); + unset($args[0]); + + if($db && $db->connected) { + $stmt = @vsprintf($sql,$args); // Disabled warnings + if($stmt === false) + logger('dba: vsprintf error: ' . print_r(debug_backtrace(),true), LOGGER_DEBUG); + $db->q("SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;"); + $retval = $db->q($stmt); + $db->q("COMMIT;"); + return $retval; + } + + /** + * + * This will happen occasionally trying to store the + * session data after abnormal program termination + * + */ + logger('dba: no database: ' . print_r($args,true)); + return false; + +} + /** * * Raw db query, no arguments diff --git a/include/dbm.php b/include/dbm.php index 72d309b22e..812989072b 100644 --- a/include/dbm.php +++ b/include/dbm.php @@ -43,6 +43,10 @@ class dbm { * @return Whether $array is a filled array */ public static function is_result($array) { + // It could be a return value from an update statement + if (is_bool($array)) + return $array; + return (is_array($array) && count($array) > 0); } } diff --git a/include/event.php b/include/event.php index a68e1c4626..4937bd73f5 100644 --- a/include/event.php +++ b/include/event.php @@ -502,7 +502,7 @@ function event_by_id($owner_uid = 0, $event_params, $sql_extra = '') { // query for the event by event id $r = q("SELECT `event`.*, `item`.`id` AS `itemid`,`item`.`plink`, `item`.`author-name`, `item`.`author-avatar`, `item`.`author-link` FROM `event` - LEFT JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid` + STRAIGHT_JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid` WHERE `event`.`uid` = %d AND `event`.`id` = %d $sql_extra", intval($owner_uid), intval($event_params["event_id"]) @@ -535,7 +535,7 @@ function events_by_date($owner_uid = 0, $event_params, $sql_extra = '') { // query for the event by date $r = q("SELECT `event`.*, `item`.`id` AS `itemid`,`item`.`plink`, `item`.`author-name`, `item`.`author-avatar`, `item`.`author-link` FROM `event` - LEFT JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid` + STRAIGHT_JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid` WHERE `event`.`uid` = %d AND event.ignore = %d AND ((`adjust` = 0 AND (`finish` >= '%s' OR (nofinish AND start >= '%s')) AND `start` <= '%s') OR (`adjust` = 1 AND (`finish` >= '%s' OR (nofinish AND start >= '%s')) AND `start` <= '%s')) diff --git a/include/items.php b/include/items.php index 5f0187ad97..aec51ba318 100644 --- a/include/items.php +++ b/include/items.php @@ -798,43 +798,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa $current_post = $r[0]['id']; logger('item_store: created item ' . $current_post); - // Set "success_update" and "last-item" to the date of the last time we heard from this contact - // This can be used to filter for inactive contacts. - // Only do this for public postings to avoid privacy problems, since poco data is public. - // Don't set this value if it isn't from the owner (could be an author that we don't know) - - $update = (!$arr['private'] AND (($arr["author-link"] === $arr["owner-link"]) OR ($arr["parent-uri"] === $arr["uri"]))); - - // Is it a forum? Then we don't care about the rules from above - if (!$update AND ($arr["network"] == NETWORK_DFRN) AND ($arr["parent-uri"] === $arr["uri"])) { - $isforum = q("SELECT `forum` FROM `contact` WHERE `id` = %d AND `forum`", - intval($arr['contact-id'])); - if ($isforum) - $update = true; - } - - if ($update) - q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d", - dbesc($arr['received']), - dbesc($arr['received']), - intval($arr['contact-id']) - ); - - // Now do the same for the system wide contacts with uid=0 - if (!$arr['private']) { - q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d", - dbesc($arr['received']), - dbesc($arr['received']), - intval($arr['owner-id']) - ); - - if ($arr['owner-id'] != $arr['author-id']) - q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d", - dbesc($arr['received']), - dbesc($arr['received']), - intval($arr['author-id']) - ); - } + item_set_last_item($arr); } else { logger('item_store: could not locate created item'); return 0; @@ -931,6 +895,51 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa return $current_post; } +/** + * @brief Set "success_update" and "last-item" to the date of the last time we heard from this contact + * + * This can be used to filter for inactive contacts. + * Only do this for public postings to avoid privacy problems, since poco data is public. + * Don't set this value if it isn't from the owner (could be an author that we don't know) + * + * @param array $arr Contains the just posted item record + */ +function item_set_last_item($arr) { + + $update = (!$arr['private'] AND (($arr["author-link"] === $arr["owner-link"]) OR ($arr["parent-uri"] === $arr["uri"]))); + + // Is it a forum? Then we don't care about the rules from above + if (!$update AND ($arr["network"] == NETWORK_DFRN) AND ($arr["parent-uri"] === $arr["uri"])) { + $isforum = q("SELECT `forum` FROM `contact` WHERE `id` = %d AND `forum`", + intval($arr['contact-id'])); + if ($isforum) + $update = true; + } + + if ($update) + q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d", + dbesc($arr['received']), + dbesc($arr['received']), + intval($arr['contact-id']) + ); + + // Now do the same for the system wide contacts with uid=0 + if (!$arr['private']) { + q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d", + dbesc($arr['received']), + dbesc($arr['received']), + intval($arr['owner-id']) + ); + + if ($arr['owner-id'] != $arr['author-id']) + q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d", + dbesc($arr['received']), + dbesc($arr['received']), + intval($arr['author-id']) + ); + } +} + function item_body_set_hashtags(&$item) { $tags = get_tags($item["body"]); diff --git a/include/post_update.php b/include/post_update.php index d45a51b5f2..a2b8497b9b 100644 --- a/include/post_update.php +++ b/include/post_update.php @@ -16,6 +16,9 @@ function post_update() { if (!post_update_1198()) return; + + if (!post_update_1206()) + return; } /** @@ -174,13 +177,18 @@ function post_update_1198() { } // Update the thread table from the item table - q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` + $r = q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid` SET `thread`.`author-id` = `item`.`author-id`, `thread`.`owner-id` = `item`.`owner-id` WHERE `thread`.`author-id` = 0 AND `thread`.`owner-id` = 0 AND (`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)"); logger("Updated threads", LOGGER_DEBUG); + if (dbm::is_result($r)) { + set_config("system", "post_update_version", 1198); + logger("Done", LOGGER_DEBUG); + return true; + } return false; } @@ -215,4 +223,38 @@ function post_update_1198() { logger("Updated items", LOGGER_DEBUG); return false; } + +/** + * @brief update the "last-item" field in the "self" contact + * + * This field avoids cost intensive calls in the admin panel and in "nodeinfo" + * + * @return bool "true" when the job is done + */ +function post_update_1206() { + // Was the script completed? + if (get_config("system", "post_update_version") >= 1206) + return true; + + logger("Start", LOGGER_DEBUG); + $r = q("SELECT `contact`.`id`, `contact`.`last-item`, + (SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` + FROM `user` + INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`"); + + if (!dbm::is_result($r)) + return false; + + foreach ($r AS $user) { + if (!empty($user["lastitem_date"]) AND ($user["lastitem_date"] > $user["last-item"])) + q("UPDATE `contact` SET `last-item` = '%s' WHERE `id` = %d", + dbesc($user["lastitem_date"]), + intval($user["id"])); + } + + set_config("system", "post_update_version", 1206); + logger("Done", LOGGER_DEBUG); + return true; +} + ?> diff --git a/mod/admin.php b/mod/admin.php index c9ff23e2d8..644583adb9 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -282,14 +282,14 @@ function admin_page_federation(&$a) { foreach ($platforms as $p) { // get a total count for the platform, the name and version of the // highest version and the protocol tpe - $c = q('SELECT COUNT(*) AS `total`, `platform`, `network`, `version` FROM `gserver` + $c = qu('SELECT COUNT(*) AS `total`, `platform`, `network`, `version` FROM `gserver` WHERE `platform` LIKE "%s" AND `last_contact` > `last_failure` AND `version` != "" ORDER BY `version` ASC;', $p); $total = $total + $c[0]['total']; // what versions for that platform do we know at all? // again only the active nodes - $v = q('SELECT COUNT(*) AS `total`, `version` FROM `gserver` + $v = qu('SELECT COUNT(*) AS `total`, `version` FROM `gserver` WHERE `last_contact` > `last_failure` AND `platform` LIKE "%s" AND `version` != "" GROUP BY `version` ORDER BY `version`;', $p); @@ -434,17 +434,17 @@ function admin_page_summary(&$a) { logger('accounts: '.print_r($accounts,true),LOGGER_DATA); - $r = q("SELECT COUNT(`id`) AS `count` FROM `register`"); + $r = qu("SELECT COUNT(`id`) AS `count` FROM `register`"); $pending = $r[0]['count']; - $r = q("SELECT COUNT(*) AS `total` FROM `deliverq` WHERE 1"); + $r = qu("SELECT COUNT(*) AS `total` FROM `deliverq` WHERE 1"); $deliverq = (($r) ? $r[0]['total'] : 0); - $r = q("SELECT COUNT(*) AS `total` FROM `queue` WHERE 1"); + $r = qu("SELECT COUNT(*) AS `total` FROM `queue` WHERE 1"); $queue = (($r) ? $r[0]['total'] : 0); if (get_config('system','worker')) { - $r = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE 1"); + $r = qu("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE 1"); $workerqueue = (($r) ? $r[0]['total'] : 0); } else { $workerqueue = 0; @@ -1271,7 +1271,7 @@ function admin_page_users(&$a){ /* get users */ - $total = q("SELECT COUNT(*) AS `total` FROM `user` WHERE 1"); + $total = qu("SELECT COUNT(*) AS `total` FROM `user` WHERE 1"); if(count($total)) { $a->set_pager_total($total[0]['total']); $a->set_pager_itemspage(100); @@ -1306,8 +1306,7 @@ function admin_page_users(&$a){ $sql_order = "`".str_replace('.','`.`',$order)."`"; $sql_order_direction = ($order_direction==="+")?"ASC":"DESC"; - $users = q("SELECT `user`.*, `contact`.`name`, `contact`.`url`, `contact`.`micro`, `user`.`account_expired`, - (SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` + $users = qu("SELECT `user`.*, `contact`.`name`, `contact`.`url`, `contact`.`micro`, `user`.`account_expired`, `contact`.`last-item` AS `lastitem_date` FROM `user` INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self` WHERE `user`.`verified` diff --git a/mod/item.php b/mod/item.php index e9056d08cc..adc46f1208 100644 --- a/mod/item.php +++ b/mod/item.php @@ -875,6 +875,8 @@ function item_post(&$a) { $datarray["id"] = $post_id; + item_set_last_item($datarray); + // update filetags in pconfig file_tag_update_pconfig($uid,$categories_old,$categories_new,'category'); diff --git a/mod/nodeinfo.php b/mod/nodeinfo.php index 585659e5cb..0d13280294 100644 --- a/mod/nodeinfo.php +++ b/mod/nodeinfo.php @@ -185,10 +185,10 @@ function nodeinfo_cron() { } logger("cron_start"); - $users = q("SELECT `user`.`uid`, `user`.`login_date`, - (SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` + $users = qu("SELECT `user`.`uid`, `user`.`login_date`, `contact`.`last-item` FROM `user` INNER JOIN `profile` ON `profile`.`uid` = `user`.`uid` AND `profile`.`is-default` + INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self` WHERE (`profile`.`publish` OR `profile`.`net-publish`) AND `user`.`verified` AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND NOT `user`.`account_expired`"); @@ -202,11 +202,11 @@ function nodeinfo_cron() { foreach ($users AS $user) { if ((strtotime($user['login_date']) > $halfyear) OR - (strtotime($user['lastitem_date']) > $halfyear)) + (strtotime($user['last-item']) > $halfyear)) ++$active_users_halfyear; if ((strtotime($user['login_date']) > $month) OR - (strtotime($user['lastitem_date']) > $month)) + (strtotime($user['last-item']) > $month)) ++$active_users_monthly; } @@ -217,8 +217,8 @@ function nodeinfo_cron() { set_config('nodeinfo','active_users_monthly', $active_users_monthly); } - //$posts = q("SELECT COUNT(*) AS local_posts FROM `item` WHERE `wall` AND `uid` != 0 AND `id` = `parent` AND left(body, 6) != '[share'"); - $posts = q("SELECT COUNT(*) AS `local_posts` FROM `item` + //$posts = qu("SELECT COUNT(*) AS local_posts FROM `item` WHERE `wall` AND `uid` != 0 AND `id` = `parent` AND left(body, 6) != '[share'"); + $posts = qu("SELECT COUNT(*) AS `local_posts` FROM `item` INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` WHERE `contact`.`self` and `item`.`id` = `item`.`parent` AND left(body, 6) != '[share' AND `item`.`network` IN ('%s', '%s', '%s')", dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_DFRN)); @@ -232,7 +232,7 @@ function nodeinfo_cron() { logger("local_posts: ".$local_posts, LOGGER_DEBUG); - $posts = q("SELECT COUNT(*) AS `local_comments` FROM `item` + $posts = qu("SELECT COUNT(*) AS `local_comments` FROM `item` INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` WHERE `contact`.`self` and `item`.`id` != `item`.`parent` AND `item`.`network` IN ('%s', '%s', '%s')", dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_DFRN));