From f07c96ee5f932a83b571ae8caaa821877d4b75e3 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 1 Jun 2016 07:04:31 +0200 Subject: [PATCH 1/8] New class "dbm" for the database management --- include/dbm.php | 26 ++++++++++++++++++++++++++ index.php | 5 +++++ 2 files changed, 31 insertions(+) create mode 100644 include/dbm.php diff --git a/include/dbm.php b/include/dbm.php new file mode 100644 index 0000000000..f68987c235 --- /dev/null +++ b/include/dbm.php @@ -0,0 +1,26 @@ + $usage) { + if ($statelist != "") + $statelist .= ", "; + $statelist .= $state.": ".$usage; + } + return($statelist); + } +} +?> diff --git a/index.php b/index.php index 73f46cfbef..ec72771341 100644 --- a/index.php +++ b/index.php @@ -41,6 +41,7 @@ $install = ((file_exists('.htconfig.php') && filesize('.htconfig.php')) ? false */ require_once("include/dba.php"); +require_once("include/dbm.php"); if(!$install) { $db = new dba($db_host, $db_user, $db_pass, $db_data, $install); @@ -53,6 +54,10 @@ if(!$install) { load_config('config'); load_config('system'); + $processlist = dbm::processlist(); + if ($processlist != "") + logger("Processlist: ".$processlist, LOGGER_DEBUG); + $maxsysload_frontend = intval(get_config('system','maxloadavg_frontend')); if($maxsysload_frontend < 1) $maxsysload_frontend = 50; From 85a9c7d96e1435a932f7e986e53d520bb6a9bc02 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 1 Jun 2016 21:54:02 +0200 Subject: [PATCH 2/8] The poller now has the logging for processes as well. --- include/poller.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/poller.php b/include/poller.php index 3a28b177c3..714d4c537c 100644 --- a/include/poller.php +++ b/include/poller.php @@ -11,6 +11,7 @@ if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) { } require_once("boot.php"); +require_once("dbm.php"); function poller_run(&$argv, &$argc){ global $a, $db; @@ -26,6 +27,10 @@ function poller_run(&$argv, &$argc){ unset($db_host, $db_user, $db_pass, $db_data); }; + $processlist = dbm::processlist(); + if ($processlist != "") + logger("Processlist: ".$processlist, LOGGER_DEBUG); + if (poller_max_connections_reached()) return; @@ -59,6 +64,11 @@ function poller_run(&$argv, &$argc){ while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `created` LIMIT 1")) { + // Log the type of database processes + $processlist = dbm::processlist(); + if ($processlist != "") + logger("Processlist: ".$processlist, LOGGER_DEBUG); + // Constantly check the number of available database connections to let the frontend be accessible at any time if (poller_max_connections_reached()) return; From df337e57d130ceb370e5269f0052254c873b27a5 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 3 Jun 2016 23:10:23 +0200 Subject: [PATCH 3/8] Quit when processlist is too long --- include/dbm.php | 4 +++- include/poller.php | 14 ++++++++++---- index.php | 10 ++++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/include/dbm.php b/include/dbm.php index f68987c235..1e1c3706e0 100644 --- a/include/dbm.php +++ b/include/dbm.php @@ -15,12 +15,14 @@ class dbm { // updating $statelist = ""; + $processes = 0; foreach ($states AS $state => $usage) { if ($statelist != "") $statelist .= ", "; $statelist .= $state.": ".$usage; + ++$processes; } - return($statelist); + return(array("list" => $statelist, "amount" => $processes)); } } ?> diff --git a/include/poller.php b/include/poller.php index 714d4c537c..45dc332e17 100644 --- a/include/poller.php +++ b/include/poller.php @@ -28,8 +28,11 @@ function poller_run(&$argv, &$argc){ }; $processlist = dbm::processlist(); - if ($processlist != "") - logger("Processlist: ".$processlist, LOGGER_DEBUG); + if ($processlist["list"] != "") { + logger("Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); + if ($processlist["amount"] > 5) + return; + } if (poller_max_connections_reached()) return; @@ -66,8 +69,11 @@ function poller_run(&$argv, &$argc){ // Log the type of database processes $processlist = dbm::processlist(); - if ($processlist != "") - logger("Processlist: ".$processlist, LOGGER_DEBUG); + if ($processlist["amount"] != "") { + logger("Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); + if ($processlist["amount"] > 5) + return; + } // Constantly check the number of available database connections to let the frontend be accessible at any time if (poller_max_connections_reached()) diff --git a/index.php b/index.php index ec72771341..a25c6dc776 100644 --- a/index.php +++ b/index.php @@ -55,8 +55,14 @@ if(!$install) { load_config('system'); $processlist = dbm::processlist(); - if ($processlist != "") - logger("Processlist: ".$processlist, LOGGER_DEBUG); + if ($processlist["list"] != "") { + logger("Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); + + // More than 20 running database processes? + // The system is too busy, so quit. + if ($processlist["amount"] > 20) + system_unavailable(); + } $maxsysload_frontend = intval(get_config('system','maxloadavg_frontend')); if($maxsysload_frontend < 1) From b45772ba6e340ce400e9f94e986eeb90d10c23aa Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 4 Jun 2016 11:04:26 +0200 Subject: [PATCH 4/8] Configurable amount of maximum processes --- include/dbm.php | 7 ++++--- include/poller.php | 22 ++++++++++++++++++---- index.php | 23 ++++++++++++++--------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/include/dbm.php b/include/dbm.php index 1e1c3706e0..d588b2f76a 100644 --- a/include/dbm.php +++ b/include/dbm.php @@ -4,23 +4,24 @@ class dbm { $r = q("SHOW PROCESSLIST"); $s = array(); + $processes = 0; $states = array(); foreach ($r AS $process) { $state = trim($process["State"]); - if (!in_array($state, array("", "init", "statistics"))) + if (!in_array($state, array("", "init", "statistics"))) { ++$states[$state]; + ++$processes; + } } // query end // Sending data // updating $statelist = ""; - $processes = 0; foreach ($states AS $state => $usage) { if ($statelist != "") $statelist .= ", "; $statelist .= $state.": ".$usage; - ++$processes; } return(array("list" => $statelist, "amount" => $processes)); } diff --git a/include/poller.php b/include/poller.php index 45dc332e17..9e7080e642 100644 --- a/include/poller.php +++ b/include/poller.php @@ -29,9 +29,16 @@ function poller_run(&$argv, &$argc){ $processlist = dbm::processlist(); if ($processlist["list"] != "") { - logger("Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); - if ($processlist["amount"] > 5) + logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); + + $max_processes = get_config('system', 'max_processes_backend'); + if (intval($max_processes) == 0) + $max_processes = 5; + + if ($processlist["amount"] > $max_processes) { + logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG); return; + } } if (poller_max_connections_reached()) @@ -70,9 +77,16 @@ function poller_run(&$argv, &$argc){ // Log the type of database processes $processlist = dbm::processlist(); if ($processlist["amount"] != "") { - logger("Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); - if ($processlist["amount"] > 5) + logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); + + $max_processes = get_config('system', 'max_processes_backend'); + if (intval($max_processes) == 0) + $max_processes = 5; + + if ($processlist["amount"] > $max_processes) { + logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG); return; + } } // Constantly check the number of available database connections to let the frontend be accessible at any time diff --git a/index.php b/index.php index a25c6dc776..17258fd0ab 100644 --- a/index.php +++ b/index.php @@ -45,7 +45,7 @@ require_once("include/dbm.php"); if(!$install) { $db = new dba($db_host, $db_user, $db_pass, $db_data, $install); - unset($db_host, $db_user, $db_pass, $db_data); + unset($db_host, $db_user, $db_pass, $db_data); /** * Load configs from db. Overwrite configs from .htconfig.php @@ -56,12 +56,17 @@ if(!$install) { $processlist = dbm::processlist(); if ($processlist["list"] != "") { - logger("Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); - // More than 20 running database processes? - // The system is too busy, so quit. - if ($processlist["amount"] > 20) - system_unavailable(); + logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); + + $max_processes = get_config('system', 'max_processes_frontend'); + if (intval($max_processes) == 0) + $max_processes = 20; + + if ($processlist["amount"] > $max_processes) { + logger("Processcheck: Maximum number of processes for frontend tasks (".$max_processes.") reached.", LOGGER_DEBUG); + return; + } } $maxsysload_frontend = intval(get_config('system','maxloadavg_frontend')); @@ -453,9 +458,9 @@ if($a->is_mobile || $a->is_tablet) { $link = 'toggle_mobile?off=1&address=' . curPageURL(); } $a->page['footer'] = replace_macros(get_markup_template("toggle_mobile_footer.tpl"), array( - '$toggle_link' => $link, - '$toggle_text' => t('toggle mobile') - )); + '$toggle_link' => $link, + '$toggle_text' => t('toggle mobile') + )); } /** From 0518f004c2f10d67a010944a7db6a815c4f22db1 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 4 Jun 2016 12:20:09 +0200 Subject: [PATCH 5/8] removed duplicated functions --- include/poller.php | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/include/poller.php b/include/poller.php index 9e7080e642..a391ea8c75 100644 --- a/include/poller.php +++ b/include/poller.php @@ -27,14 +27,14 @@ function poller_run(&$argv, &$argc){ unset($db_host, $db_user, $db_pass, $db_data); }; + $max_processes = get_config('system', 'max_processes_backend'); + if (intval($max_processes) == 0) + $max_processes = 5; + $processlist = dbm::processlist(); if ($processlist["list"] != "") { logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); - $max_processes = get_config('system', 'max_processes_backend'); - if (intval($max_processes) == 0) - $max_processes = 5; - if ($processlist["amount"] > $max_processes) { logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG); return; @@ -79,10 +79,6 @@ function poller_run(&$argv, &$argc){ if ($processlist["amount"] != "") { logger("Processcheck: Processes: ".$processlist["amount"]." - Processlist: ".$processlist["list"], LOGGER_DEBUG); - $max_processes = get_config('system', 'max_processes_backend'); - if (intval($max_processes) == 0) - $max_processes = 5; - if ($processlist["amount"] > $max_processes) { logger("Processcheck: Maximum number of processes for backend tasks (".$max_processes.") reached.", LOGGER_DEBUG); return; From e33defbe3e8e3933f81ab9f0e2086a4c41d403f3 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 4 Jun 2016 12:23:12 +0200 Subject: [PATCH 6/8] Added documentation --- doc/htconfig.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/htconfig.md b/doc/htconfig.md index 77e63671ab..4f4e34d07c 100644 --- a/doc/htconfig.md +++ b/doc/htconfig.md @@ -38,6 +38,8 @@ line to your .htconfig.php: * max_connections_level - The maximum level of connections that are allowed to let the poller start. It is a percentage value. Default value is 75. * max_contact_queue - Default value is 500. * max_batch_queue - Default value is 1000. +* max_processes_backend - Maximum number of concurrent database processes for background tasks. Default value is 5. +* max_processes_frontend'] - Maximum number of concurrent database processes for foreground tasks. Default value is 20. * no_oembed (Boolean) - Don't use OEmbed to fetch more information about a link. * no_oembed_rich_content (Boolean) - Don't show the rich content (e.g. embedded PDF). * no_smilies (Boolean) - Don't show smilies. From 5420c98ff0498d4a1616455a4b34d95dd56b26bd Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 4 Jun 2016 12:30:24 +0200 Subject: [PATCH 7/8] removed some copy&paste fragments --- doc/htconfig.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/htconfig.md b/doc/htconfig.md index 4f4e34d07c..2435da2baa 100644 --- a/doc/htconfig.md +++ b/doc/htconfig.md @@ -39,7 +39,7 @@ line to your .htconfig.php: * max_contact_queue - Default value is 500. * max_batch_queue - Default value is 1000. * max_processes_backend - Maximum number of concurrent database processes for background tasks. Default value is 5. -* max_processes_frontend'] - Maximum number of concurrent database processes for foreground tasks. Default value is 20. +* max_processes_frontend - Maximum number of concurrent database processes for foreground tasks. Default value is 20. * no_oembed (Boolean) - Don't use OEmbed to fetch more information about a link. * no_oembed_rich_content (Boolean) - Don't show the rich content (e.g. embedded PDF). * no_smilies (Boolean) - Don't show smilies. From 3d27ebcb341dd2c1ed8613469362937c51727d5e Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 6 Jun 2016 06:29:03 +0200 Subject: [PATCH 8/8] Added documentation --- include/dbm.php | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/include/dbm.php b/include/dbm.php index d588b2f76a..3f936947d8 100644 --- a/include/dbm.php +++ b/include/dbm.php @@ -1,5 +1,16 @@ List of processes, separated in their different states + * 'amount' => Number of concurrent database processes + */ public static function processlist() { $r = q("SHOW PROCESSLIST"); $s = array(); @@ -8,14 +19,13 @@ class dbm { $states = array(); foreach ($r AS $process) { $state = trim($process["State"]); + + // Filter out all idle processes if (!in_array($state, array("", "init", "statistics"))) { ++$states[$state]; ++$processes; } } - // query end - // Sending data - // updating $statelist = ""; foreach ($states AS $state => $usage) {