mirror of
https://github.com/friendica/friendica
synced 2025-04-27 13:50:12 +00:00
Merge pull request #5157 from annando/daemon
Workers can now be started exclusively from the daemon and other workers
This commit is contained in:
Hypolite Petovan
GitHub
commit
973ece09c5
6 changed files with 236 additions and 127 deletions
|
|
@ -53,10 +53,15 @@ class Worker
|
|||
// We now start the process. This is done after the load check since this could increase the load.
|
||||
self::startProcess();
|
||||
|
||||
// The daemon doesn't need to fork new workers anymore, since we started a process
|
||||
if (Config::get('system', 'worker_daemon_mode', false)) {
|
||||
self::IPCSetJobState(false);
|
||||
}
|
||||
|
||||
// Kill stale processes every 5 minutes
|
||||
$last_cleanup = Config::get('system', 'poller_last_cleaned', 0);
|
||||
$last_cleanup = Config::get('system', 'worker_last_cleaned', 0);
|
||||
if (time() > ($last_cleanup + 300)) {
|
||||
Config::set('system', 'poller_last_cleaned', time());
|
||||
Config::set('system', 'worker_last_cleaned', time());
|
||||
self::killStaleWorkers();
|
||||
}
|
||||
|
||||
|
|
@ -108,16 +113,16 @@ class Worker
|
|||
}
|
||||
|
||||
// If possible we will fetch new jobs for this worker
|
||||
if (!$refetched && Lock::set('poller_worker_process', 0)) {
|
||||
if (!$refetched && Lock::set('worker_process', 0)) {
|
||||
$stamp = (float)microtime(true);
|
||||
$refetched = self::findWorkerProcesses($passing_slow);
|
||||
self::$db_duration += (microtime(true) - $stamp);
|
||||
Lock::remove('poller_worker_process');
|
||||
Lock::remove('worker_process');
|
||||
}
|
||||
}
|
||||
|
||||
// To avoid the quitting of multiple workers only one worker at a time will execute the check
|
||||
if (Lock::set('poller_worker', 0)) {
|
||||
if (Lock::set('worker', 0)) {
|
||||
$stamp = (float)microtime(true);
|
||||
// Count active workers and compare them with a maximum value that depends on the load
|
||||
if (self::tooMuchWorkers()) {
|
||||
|
|
@ -130,7 +135,7 @@ class Worker
|
|||
logger('Memory limit reached, quitting.', LOGGER_DEBUG);
|
||||
return;
|
||||
}
|
||||
Lock::remove('poller_worker');
|
||||
Lock::remove('worker');
|
||||
self::$db_duration += (microtime(true) - $stamp);
|
||||
}
|
||||
|
||||
|
|
@ -140,6 +145,9 @@ class Worker
|
|||
return;
|
||||
}
|
||||
}
|
||||
if (Config::get('system', 'worker_daemon_mode', false)) {
|
||||
self::IPCSetJobState(false);
|
||||
}
|
||||
logger("Couldn't select a workerqueue entry, quitting.", LOGGER_DEBUG);
|
||||
}
|
||||
|
||||
|
|
@ -244,7 +252,7 @@ class Worker
|
|||
|
||||
$stamp = (float)microtime(true);
|
||||
if (dba::update('workerqueue', ['done' => true], ['id' => $queue["id"]])) {
|
||||
Config::set('system', 'last_poller_execution', DateTimeFormat::utcNow());
|
||||
Config::set('system', 'last_worker_execution', DateTimeFormat::utcNow());
|
||||
}
|
||||
self::$db_duration = (microtime(true) - $stamp);
|
||||
|
||||
|
|
@ -285,7 +293,7 @@ class Worker
|
|||
|
||||
$stamp = (float)microtime(true);
|
||||
if (dba::update('workerqueue', ['done' => true], ['id' => $queue["id"]])) {
|
||||
Config::set('system', 'last_poller_execution', DateTimeFormat::utcNow());
|
||||
Config::set('system', 'last_worker_execution', DateTimeFormat::utcNow());
|
||||
}
|
||||
self::$db_duration = (microtime(true) - $stamp);
|
||||
} else {
|
||||
|
|
@ -851,6 +859,11 @@ class Worker
|
|||
dba::update('workerqueue', ['executed' => DateTimeFormat::utcNow(), 'pid' => $mypid], $ids);
|
||||
}
|
||||
|
||||
// The daemon doesn't need to fork new workers anymore, since we are inside the worker
|
||||
if (Config::get('system', 'worker_daemon_mode', false)) {
|
||||
self::IPCSetJobState(false);
|
||||
}
|
||||
|
||||
return $found;
|
||||
}
|
||||
|
||||
|
|
@ -873,7 +886,7 @@ class Worker
|
|||
dba::close($r);
|
||||
|
||||
$stamp = (float)microtime(true);
|
||||
if (!Lock::set('poller_worker_process')) {
|
||||
if (!Lock::set('worker_process')) {
|
||||
return false;
|
||||
}
|
||||
self::$lock_duration = (microtime(true) - $stamp);
|
||||
|
|
@ -882,7 +895,7 @@ class Worker
|
|||
$found = self::findWorkerProcesses($passing_slow);
|
||||
self::$db_duration += (microtime(true) - $stamp);
|
||||
|
||||
Lock::remove('poller_worker_process');
|
||||
Lock::remove('worker_process');
|
||||
|
||||
if ($found) {
|
||||
$r = dba::select('workerqueue', [], ['pid' => getmypid(), 'done' => false]);
|
||||
|
|
@ -1076,14 +1089,20 @@ class Worker
|
|||
return true;
|
||||
}
|
||||
|
||||
// We tell the daemon that a new job entry exists
|
||||
if (Config::get('system', 'worker_daemon_mode', false)) {
|
||||
self::IPCSetJobState(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
// If there is a lock then we don't have to check for too much worker
|
||||
if (!Lock::set('poller_worker', 0)) {
|
||||
if (!Lock::set('worker', 0)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// If there are already enough workers running, don't fork another one
|
||||
$quit = self::tooMuchWorkers();
|
||||
Lock::remove('poller_worker');
|
||||
Lock::remove('worker');
|
||||
|
||||
if ($quit) {
|
||||
return true;
|
||||
|
|
@ -1121,4 +1140,33 @@ class Worker
|
|||
{
|
||||
return Process::deleteByPid();
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the flag if some job is waiting
|
||||
*
|
||||
* @brief Set the flag if some job is waiting
|
||||
* @param boolean $jobs Is there a waiting job?
|
||||
*/
|
||||
public static function IPCSetJobState($jobs)
|
||||
{
|
||||
dba::update('worker-ipc', ['jobs' => $jobs], ['key' => 1], true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if some worker job waits to be executed
|
||||
*
|
||||
* @brief Checks if some worker job waits to be executed
|
||||
* @return bool
|
||||
*/
|
||||
public static function IPCJobsExists()
|
||||
{
|
||||
$row = dba::selectFirst('worker-ipc', ['jobs'], ['key' => 1]);
|
||||
|
||||
// When we don't have a row, no job is running
|
||||
if (!DBM::is_result($row)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (bool)$row['jobs'];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ class DBStructure
|
|||
echo "--\n";
|
||||
echo "-- TABLE $name\n";
|
||||
echo "--\n";
|
||||
self::createTable($name, $structure['fields'], true, false, $structure["indexes"]);
|
||||
self::createTable($name, $structure, true, false);
|
||||
|
||||
echo "\n";
|
||||
}
|
||||
|
|
@ -251,7 +251,7 @@ class DBStructure
|
|||
$is_unique = false;
|
||||
$temp_name = $name;
|
||||
if (!isset($database[$name])) {
|
||||
$r = self::createTable($name, $structure["fields"], $verbose, $action, $structure['indexes']);
|
||||
$r = self::createTable($name, $structure, $verbose, $action);
|
||||
if (!DBM::is_result($r)) {
|
||||
$errors .= self::printUpdateError($name);
|
||||
}
|
||||
|
|
@ -378,6 +378,18 @@ class DBStructure
|
|||
}
|
||||
}
|
||||
|
||||
if (isset($database[$name]["table_status"]["Engine"]) && isset($structure['engine'])) {
|
||||
if ($database[$name]["table_status"]["Engine"] != $structure['engine']) {
|
||||
$sql2 = "ENGINE = '".dbesc($structure['engine'])."'";
|
||||
|
||||
if ($sql3 == "") {
|
||||
$sql3 = "ALTER" . $ignore . " TABLE `".$temp_name."` ".$sql2;
|
||||
} else {
|
||||
$sql3 .= ", ".$sql2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($database[$name]["table_status"]["Collation"])) {
|
||||
if ($database[$name]["table_status"]["Collation"] != 'utf8mb4_general_ci') {
|
||||
$sql2 = "DEFAULT COLLATE utf8mb4_general_ci";
|
||||
|
|
@ -554,20 +566,22 @@ class DBStructure
|
|||
return($fieldstruct);
|
||||
}
|
||||
|
||||
private static function createTable($name, $fields, $verbose, $action, $indexes=null) {
|
||||
private static function createTable($name, $structure, $verbose, $action) {
|
||||
$r = true;
|
||||
|
||||
$engine = "";
|
||||
$comment = "";
|
||||
$sql_rows = [];
|
||||
$primary_keys = [];
|
||||
foreach ($fields AS $fieldname => $field) {
|
||||
foreach ($structure["fields"] AS $fieldname => $field) {
|
||||
$sql_rows[] = "`".dbesc($fieldname)."` ".self::FieldCommand($field);
|
||||
if (x($field,'primary') && $field['primary']!='') {
|
||||
$primary_keys[] = $fieldname;
|
||||
}
|
||||
}
|
||||
|
||||
if (!is_null($indexes)) {
|
||||
foreach ($indexes AS $indexname => $fieldnames) {
|
||||
if (!is_null($structure["indexes"])) {
|
||||
foreach ($structure["indexes"] AS $indexname => $fieldnames) {
|
||||
$sql_index = self::createIndex($indexname, $fieldnames, "");
|
||||
if (!is_null($sql_index)) {
|
||||
$sql_rows[] = $sql_index;
|
||||
|
|
@ -575,9 +589,18 @@ class DBStructure
|
|||
}
|
||||
}
|
||||
|
||||
if (!is_null($structure["engine"])) {
|
||||
$engine = " ENGINE=" . $structure["engine"];
|
||||
}
|
||||
|
||||
if (!is_null($structure["comment"])) {
|
||||
$comment = " COMMENT='" . dbesc($structure["comment"]) . "'";
|
||||
}
|
||||
|
||||
$sql = implode(",\n\t", $sql_rows);
|
||||
|
||||
$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n\t", dbesc($name)).$sql."\n) DEFAULT COLLATE utf8mb4_general_ci";
|
||||
$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n\t", dbesc($name)).$sql.
|
||||
"\n)" . $engine . " DEFAULT COLLATE utf8mb4_general_ci" . $comment;
|
||||
if ($verbose) {
|
||||
echo $sql.";\n";
|
||||
}
|
||||
|
|
@ -1797,6 +1820,18 @@ class DBStructure
|
|||
"PRIMARY" => ["uid", "iid"],
|
||||
]
|
||||
];
|
||||
$database["worker-ipc"] = [
|
||||
"comment" => "Inter process communication between the frontend and the worker",
|
||||
"fields" => [
|
||||
"key" => ["type" => "int", "not null" => "1", "primary" => "1", "comment" => ""],
|
||||
"jobs" => ["type" => "boolean", "comment" => "Flag for outstanding jobs"],
|
||||
],
|
||||
"indexes" => [
|
||||
"PRIMARY" => ["key"],
|
||||
],
|
||||
"engine" => "MEMORY",
|
||||
];
|
||||
|
||||
$database["workerqueue"] = [
|
||||
"comment" => "Background tasks queue entries",
|
||||
"fields" => [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue