Merge pull request #10309 from fabrixxm/feature/advanced-logsview

Display structured logs in admin

src/DI.php

@@ -394,6 +394,14 @@ abstract class DI
 		return self::$dice->create(Model\Storage\IWritableStorage::class);
 	}
 
+	/**
+	 * @return Model\Log\ParsedLogIterator
+	 */
+	public static function parsedLogIterator()
+	{
+		return self::$dice->create(Model\Log\ParsedLogIterator::class);
+	}
+
 	//
 	// "Network" namespace
 	//

src/Model/Log/ParsedLogIterator.php

@@ -0,0 +1,220 @@
+<?php
+/**
+ * @copyright Copyright (C) 2021, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Log;
+
+use Friendica\Util\ReversedFileReader;
+use Friendica\Object\Log\ParsedLogLine;
+
+/**
+ * An iterator which returns `\Friendica\Objec\Log\ParsedLogLine` instances
+ *
+ * Uses `\Friendica\Util\ReversedFileReader` to fetch log lines
+ * from newest to oldest.
+ */
+class ParsedLogIterator implements \Iterator
+{
+	/** @var \Iterator */
+	private $reader;
+
+	/** @var ParsedLogLine current iterator value*/
+	private $value = null;
+
+	/** @var int max number of lines to read */
+	private $limit = 0;
+
+	/** @var array filters per column */
+	private $filters = [];
+
+	/** @var string search term */
+	private $search = "";
+
+
+	/**
+	 * @param ReversedFileReader $reader
+	 */
+	public function __construct(ReversedFileReader $reader)
+	{
+		$this->reader = $reader;
+	}
+
+	/**
+	 * @param string $filename	File to open
+	 * @return $this
+	 */
+	public function open(string $filename)
+	{
+		$this->reader->open($filename);
+		return $this;
+	}
+
+	/**
+	 * @param int $limit		Max num of lines to read
+	 * @return $this
+	 */
+	public function withLimit(int $limit)
+	{
+		$this->limit = $limit;
+		return $this;
+	}
+
+	/**
+	 * @param array $filters		filters per column
+	 * @return $this
+	 */
+	public function withFilters(array $filters)
+	{
+		$this->filters = $filters;
+		return $this;
+	}
+
+	/**
+	 * @param string $search	string to search to filter lines
+	 * @return $this
+	 */
+	public function withSearch(string $search)
+	{
+		$this->search = $search;
+		return $this;
+	}
+
+	/**
+	 * Check if parsed log line match filters.
+	 * Always match if no filters are set.
+	 *
+	 * @param ParsedLogLine $parsedlogline
+	 * @return bool
+	 */
+	private function filter($parsedlogline)
+	{
+		$match = true;
+		foreach ($this->filters as $filter => $filtervalue) {
+			switch ($filter) {
+				case "level":
+					$match = $match && ($parsedlogline->level == strtoupper($filtervalue));
+					break;
+				case "context":
+					$match = $match && ($parsedlogline->context == $filtervalue);
+					break;
+			}
+		}
+		return $match;
+	}
+
+	/**
+	 * Check if parsed log line match search.
+	 * Always match if no search query is set.
+	 *
+	 * @param ParsedLogLine $parsedlogline
+	 * @return bool
+	 */
+	private function search($parsedlogline)
+	{
+		if ($this->search != "") {
+			return strstr($parsedlogline->logline, $this->search) !== false;
+		}
+		return true;
+	}
+
+	/**
+	 * Read a line from reader and parse.
+	 * Returns null if limit is reached or the reader is invalid.
+	 *
+	 * @param ParsedLogLine $parsedlogline
+	 * @return ?ParsedLogLine
+	 */
+	private function read()
+	{
+		$this->reader->next();
+		if ($this->limit > 0 && $this->reader->key() > $this->limit || !$this->reader->valid()) {
+			return null;
+		}
+
+		$line = $this->reader->current();
+		return new ParsedLogLine($this->reader->key(), $line);
+	}
+
+
+	/**
+	 * Fetch next parsed log line which match with filters or search and
+	 * set it as current iterator value.
+	 *
+	 * @see Iterator::next()
+	 * @return void
+	 */
+	public function next()
+	{
+		$parsed = $this->read();
+
+		while (is_null($parsed) == false && !($this->filter($parsed) && $this->search($parsed))) {
+			$parsed = $this->read();
+		}
+		$this->value = $parsed;
+	}
+
+
+	/**
+	 * Rewind the iterator to the first matching log line
+	 *
+	 * @see Iterator::rewind()
+	 * @return void
+	 */
+	public function rewind()
+	{
+		$this->value = null;
+		$this->reader->rewind();
+		$this->next();
+	}
+
+	/**
+	 * Return current parsed log line number
+	 *
+	 * @see Iterator::key()
+	 * @see ReversedFileReader::key()
+	 * @return int
+	 */
+	public function key()
+	{
+		return $this->reader->key();
+	}
+
+	/**
+	 * Return current iterator value
+	 *
+	 * @see Iterator::current()
+	 * @return ?ParsedLogLing
+	 */
+	public function current()
+	{
+		return $this->value;
+	}
+
+	/**
+	 * Checks if current iterator value is valid, that is, not null
+	 *
+	 * @see Iterator::valid()
+	 * @return bool
+	 */
+	public function valid()
+	{
+		return ! is_null($this->value);
+	}
+}

src/Module/Admin/Logs/View.php

@@ -21,50 +21,74 @@
 
 namespace Friendica\Module\Admin\Logs;
 
-use Friendica\Core\Renderer;
 use Friendica\DI;
+use Friendica\Core\Renderer;
+use Friendica\Core\Theme;
 use Friendica\Module\BaseAdmin;
-use Friendica\Util\Strings;
+use Friendica\Model\Log\ParsedLogIterator;
+use Psr\Log\LogLevel;
 
 class View extends BaseAdmin
 {
+	const LIMIT = 500;
+
 	public static function content(array $parameters = [])
 	{
 		parent::content($parameters);
 
 		$t = Renderer::getMarkupTemplate('admin/logs/view.tpl');
+		DI::page()->registerFooterScript(Theme::getPathForFile('js/module/admin/logs/view.js'));
+
 		$f     = DI::config()->get('system', 'logfile');
-		$data = '';
+		$data  = null;
+		$error = null;
+
+		$search = $_GET['q'] ?? '';
+
+		$filters_valid_values = [
+			'level' => [
+				'',
+				LogLevel::CRITICAL,
+				LogLevel::ERROR,
+				LogLevel::WARNING,
+				LogLevel::NOTICE,
+				LogLevel::INFO,
+				LogLevel::DEBUG,
+			],
+			'context' => ['', 'index', 'worker'],
+		];
+		$filters = [
+			'level'   => $_GET['level'] ?? '',
+			'context' => $_GET['context'] ?? '',
+		];
+		foreach ($filters as $k => $v) {
+			if ($v == '' || !in_array($v, $filters_valid_values[$k])) {
+				unset($filters[$k]);
+			}
+		}
 
 		if (!file_exists($f)) {
-			$data = DI::l10n()->t('Error trying to open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s exist and is readable.', $f);
+			$error = DI::l10n()->t('Error trying to open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s exist and is readable.', $f);
 		} else {
-			$fp = fopen($f, 'r');
+			try {
-			if (!$fp) {
+				$data = DI::parsedLogIterator()
-				$data = DI::l10n()->t('Couldn\'t open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s is readable.', $f);
+						->open($f)
-			} else {
+						->withLimit(self::LIMIT)
-				$fstat = fstat($fp);
+						->withFilters($filters)
-				$size = $fstat['size'];
+						->withSearch($search);
-				if ($size != 0) {
+			} catch (Exception $e) {
-					if ($size > 5000000 || $size < 0) {
+				$error = DI::l10n()->t('Couldn\'t open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s is readable.', $f);
-						$size = 5000000;
-					}
-					$seek = fseek($fp, 0 - $size, SEEK_END);
-					if ($seek === 0) {
-						$data = Strings::escapeHtml(fread($fp, $size));
-						while (!feof($fp)) {
-							$data .= Strings::escapeHtml(fread($fp, 4096));
-						}
-					}
-				}
-				fclose($fp);
 			}
 		}
 		return Renderer::replaceMacros($t, [
 			'$title'         => DI::l10n()->t('Administration'),
 			'$page'          => DI::l10n()->t('View Logs'),
 			'$data'          => $data,
-			'$logname' => DI::config()->get('system', 'logfile')
+			'$q'             => $search,
+			'$filters'       => $filters,
+			'$filtersvalues' => $filters_valid_values,
+			'$error'         => $error,
+			'$logname'       => DI::config()->get('system', 'logfile'),
 		]);
 	}
 }

src/Object/Log/ParsedLogLine.php

@@ -0,0 +1,164 @@
+<?php
+/**
+ * @copyright Copyright (C) 2021, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Object\Log;
+
+/**
+ * Parse a log line and offer some utility methods
+ */
+class ParsedLogLine
+{
+	const REGEXP = '/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^ ]*) (\w+) \[(\w*)\]: (.*)/';
+
+	/** @var int */
+	public $id = 0;
+
+	/** @var string */
+	public $date = null;
+
+	/** @var string */
+	public $context = null;
+
+	/** @var string */
+	public $level = null;
+
+	/** @var string */
+	public $message = null;
+
+	/** @var string */
+	public $data = null;
+
+	/** @var string */
+	public $source = null;
+
+	/** @var string */
+	public $logline;
+
+	/**
+	 * @param int line id
+	 * @param string $logline Source log line to parse
+	 */
+	public function __construct(int $id, string $logline)
+	{
+		$this->id = $id;
+		$this->parse($logline);
+	}
+
+	private function parse($logline)
+	{
+		$this->logline = $logline;
+
+		// if data is empty is serialized as '[]'. To ease the parsing
+		// let's replace it with '{""}'. It will be replaced by null later
+		$logline = str_replace(' [] - {', ' {""} - {', $logline);
+
+
+		if (strstr($logline, ' - {') === false) {
+			// the log line is not well formed
+			$jsonsource = null;
+		} else {
+			// here we hope that there will not be the string ' - {' inside the $jsonsource value
+			list($logline, $jsonsource) = explode(' - {', $logline);
+			$jsonsource                 = '{' . $jsonsource;
+		}
+
+		$jsondata = null;
+		if (strpos($logline, '{"') > 0) {
+			list($logline, $jsondata) = explode('{"', $logline, 2);
+
+			$jsondata = '{"' . $jsondata;
+		}
+
+		preg_match(self::REGEXP, $logline, $matches);
+
+		if (count($matches) == 0) {
+			// regexp not matching
+			$this->message = $this->logline;
+		} else {
+			$this->date    = $matches[1];
+			$this->context = $matches[2];
+			$this->level   = $matches[3];
+			$this->message = $matches[4];
+			$this->data    = $jsondata == '{""}' ? null : $jsondata;
+			$this->source  = $jsonsource;
+			$this->tryfixjson();
+		}
+
+		$this->message = trim($this->message);
+	}
+
+	/**
+	 * Fix message / data split
+	 *
+	 * In log boundary between message and json data is not specified.
+	 * If message  contains '{' the parser thinks there starts the json data.
+	 * This method try to parse the found json and if it fails, search for next '{'
+	 * in json data and retry
+	 */
+	private function tryfixjson()
+	{
+		if (is_null($this->data) || $this->data == '') {
+			return;
+		}
+		try {
+			$d = json_decode($this->data, true, 512, JSON_THROW_ON_ERROR);
+		} catch (\JsonException $e) {
+			// try to find next { in $str and move string before to 'message'
+
+			$pos = strpos($this->data, '{', 1);
+			if ($pos === false) {
+				$this->message .= $this->data;
+				$this->data = null;
+				return;
+			}
+
+			$this->message .= substr($this->data, 0, $pos);
+			$this->data = substr($this->data, $pos);
+			$this->tryfixjson();
+		}
+	}
+
+	/**
+	 * Return decoded `data` as array suitable for template
+	 *
+	 * @return array
+	 */
+	public function getData()
+	{
+		$data = json_decode($this->data, true);
+		if ($data) {
+			foreach ($data as $k => $v) {
+				$data[$k] = print_r($v, true);
+			}
+		}
+		return $data;
+	}
+
+	/**
+	 * Return decoded `source` as array suitable for template
+	 *
+	 * @return array
+	 */
+	public function getSource()
+	{
+		return json_decode($this->source, true);
+	}
+}

src/Util/ReversedFileReader.php

@@ -0,0 +1,166 @@
+<?php
+/**
+ * @copyright Copyright (C) 2021, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Util;
+
+/**
+ * An iterator which returns lines from file in reversed order
+ *
+ * original code https://stackoverflow.com/a/10494801
+ */
+class ReversedFileReader implements \Iterator
+{
+	const BUFFER_SIZE = 4096;
+	const SEPARATOR   = "\n";
+
+	/** @var resource */
+	private $fh = null;
+
+	/** @var int */
+	private $filesize = -1;
+
+	/** @var int */
+	private $pos = -1;
+
+	/** @var array */
+	private $buffer = null;
+
+	/** @var int */
+	private $key = -1;
+
+	/** @var string */
+	private $value = null;
+
+	/**
+	 * Open $filename for read and reset iterator
+	 *
+	 * @param string $filename	File to open
+	 * @return $this
+	 */
+	public function open(string $filename)
+	{
+		$this->fh = fopen($filename, 'r');
+		if (!$this->fh) {
+			// this should use a custom exception.
+			throw \Exception("Unable to open $filename");
+		}
+		$this->filesize = filesize($filename);
+		$this->pos      = -1;
+		$this->buffer   = null;
+		$this->key      = -1;
+		$this->value    = null;
+		return $this;
+	}
+
+	/**
+	 * Read $size bytes behind last position
+	 *
+	 * @return string
+	 */
+	private function _read($size)
+	{
+		$this->pos -= $size;
+		fseek($this->fh, $this->pos);
+		return fread($this->fh, $size);
+	}
+
+	/**
+	 * Read next line from end of file
+	 * Return null if no lines are left to read
+	 *
+	 * @return ?string
+	 */
+	private function _readline()
+	{
+		$buffer = & $this->buffer;
+		while (true) {
+			if ($this->pos == 0) {
+				return array_pop($buffer);
+			}
+			if (count($buffer) > 1) {
+				return array_pop($buffer);
+			}
+			$buffer = explode(self::SEPARATOR, $this->_read(self::BUFFER_SIZE) . $buffer[0]);
+		}
+	}
+
+	/**
+	 * Fetch next line from end and set it as current iterator value.
+	 *
+	 * @see Iterator::next()
+	 * @return void
+	 */
+	public function next()
+	{
+		++$this->key;
+		$this->value = $this->_readline();
+	}
+
+	/**
+	 * Rewind iterator to the first line at the end of file
+	 *
+	 * @see Iterator::rewind()
+	 * @return void
+	 */
+	public function rewind()
+	{
+		if ($this->filesize > 0) {
+			$this->pos    = $this->filesize;
+			$this->value  = null;
+			$this->key    = -1;
+			$this->buffer = explode(self::SEPARATOR, $this->_read($this->filesize % self::BUFFER_SIZE ?: self::BUFFER_SIZE));
+			$this->next();
+		}
+	}
+
+	/**
+	 * Return current line number, starting from zero at the end of file
+	 *
+	 * @see Iterator::key()
+	 * @return int
+	 */
+	public function key()
+	{
+		return $this->key;
+	}
+
+	/**
+	 * Return current line
+	 *
+	 * @see Iterator::current()
+	 * @return string
+	 */
+	public function current()
+	{
+		return $this->value;
+	}
+
+	/**
+	 * Checks if current iterator value is valid, that is, we readed all lines in files
+	 *
+	 * @see Iterator::valid()
+	 * @return bool
+	 */
+	public function valid()
+	{
+		return ! is_null($this->value);
+	}
+}

static/dependencies.config.php

@@ -46,6 +46,7 @@ use Friendica\Database\Database;
 use Friendica\Factory;
 use Friendica\Model\Storage\IWritableStorage;
 use Friendica\Model\User\Cookie;
+use Friendica\Model\Log\ParsedLogIterator;
 use Friendica\Network;
 use Friendica\Util;
 use Psr\Log\LoggerInterface;
@@ -227,4 +228,9 @@ return [
 			$_SERVER
 		],
 	],
+	ParsedLogIterator::class => [
+		'constructParams' => [
+			[Dice::INSTANCE => Util\ReversedFileReader::class],
+		]
+	]
 ];

tests/datasets/log/friendica.log.txt

@@ -0,0 +1,3 @@
+2021-05-24T15:23:58Z index [INFO]: No HTTP_SIGNATURE header [] - {"file":"HTTPSignature.php","line":476,"function":"getSigner","uid":"0a3934","process_id":14826}
+2021-05-24T15:30:01Z worker [NOTICE]: Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10 {"worker_id":"ece8fc8","worker_cmd":"Cron"} - {"file":"Worker.php","line":786,"function":"tooMuchWorkers","uid":"364d3c","process_id":20754}
+2021-05-24T15:40:01Z worker [WARNING]: Spool file does does not start with "item-" {"file":".","worker_id":"560c8b6","worker_cmd":"SpoolPost"} - {"file":"SpoolPost.php","line":40,"function":"execute","uid":"fd8c37","process_id":20846}

tests/src/Model/Log/ParsedLogIteratorTest.php

@@ -0,0 +1,148 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2021, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Test\src\Object\Log;
+
+use Friendica\Util\ReversedFileReader;
+use Friendica\Model\Log\ParsedLogIterator;
+
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Parsed log iterator testing class
+ */
+class ParsedLogIteratorTest extends TestCase
+{
+	protected $pli;
+
+	public static function assertParsed($parsed, $expected_data)
+	{
+		foreach ($expected_data as $k => $v) {
+			self::assertSame($parsed->$k, $v, '"'.$k.'" does not match expectation');
+		}
+	}
+
+	protected function setUp(): void
+	{
+		$logfile = dirname(__DIR__) . '/../../datasets/log/friendica.log.txt';
+
+		$reader    = new ReversedFileReader();
+		$this->pli = new ParsedLogIterator($reader);
+		$this->pli->open($logfile);
+	}
+
+	public function testIsIterable()
+	{
+		self::assertIsIterable($this->pli);
+	}
+
+	public function testEverything()
+	{
+		self::assertCount(3, iterator_to_array($this->pli, false));
+	}
+
+	public function testLimit()
+	{
+		$this->pli->withLimit(2);
+		self::assertCount(2, iterator_to_array($this->pli, false));
+	}
+
+	public function testFilterByLevel()
+	{
+		$this->pli->withFilters(['level' => 'INFO']);
+		$pls = iterator_to_array($this->pli, false);
+		self::assertCount(1, $pls);
+		self::assertParsed(
+			$pls[0],
+			[
+				'date'    => '2021-05-24T15:23:58Z',
+				'context' => 'index',
+				'level'   => 'INFO',
+				'message' => 'No HTTP_SIGNATURE header',
+				'data'    => null,
+				'source'  => '{"file":"HTTPSignature.php","line":476,"function":"getSigner","uid":"0a3934","process_id":14826}',
+			]
+		);
+	}
+
+	public function testFilterByContext()
+	{
+		$this->pli->withFilters(['context' => 'worker']);
+		$pls = iterator_to_array($this->pli, false);
+		self::assertCount(2, $pls);
+		self::assertParsed(
+			$pls[0],
+			[
+				'date'    => '2021-05-24T15:40:01Z',
+				'context' => 'worker',
+				'level'   => 'WARNING',
+				'message' => 'Spool file does does not start with "item-"',
+				'data'    => '{"file":".","worker_id":"560c8b6","worker_cmd":"SpoolPost"}',
+				'source'  => '{"file":"SpoolPost.php","line":40,"function":"execute","uid":"fd8c37","process_id":20846}',
+			]
+		);
+	}
+
+	public function testFilterCombined()
+	{
+		$this->pli->withFilters(['level' => 'NOTICE', 'context' => 'worker']);
+		$pls = iterator_to_array($this->pli, false);
+		self::assertCount(1, $pls);
+		self::assertParsed(
+			$pls[0],
+			[
+				'date'    => '2021-05-24T15:30:01Z',
+				'context' => 'worker',
+				'level'   => 'NOTICE',
+				'message' => 'Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10',
+				'data'    => '{"worker_id":"ece8fc8","worker_cmd":"Cron"}',
+				'source'  => '{"file":"Worker.php","line":786,"function":"tooMuchWorkers","uid":"364d3c","process_id":20754}',
+			]
+		);
+	}
+
+	public function testSearch()
+	{
+		$this->pli->withSearch("maximum");
+		$pls = iterator_to_array($this->pli, false);
+		self::assertCount(1, $pls);
+		self::assertParsed(
+			$pls[0],
+			[
+				'date'    => '2021-05-24T15:30:01Z',
+				'context' => 'worker',
+				'level'   => 'NOTICE',
+				'message' => 'Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10',
+				'data'    => '{"worker_id":"ece8fc8","worker_cmd":"Cron"}',
+				'source'  => '{"file":"Worker.php","line":786,"function":"tooMuchWorkers","uid":"364d3c","process_id":20754}',
+			]
+		);
+	}
+
+	public function testFilterAndSearch()
+	{
+		$this->pli
+			->withFilters(['context' => 'worker'])
+			->withSearch("header");
+		$pls = iterator_to_array($this->pli, false);
+		self::assertCount(0, $pls);
+	}
+}

tests/src/Object/Log/ParsedLogLineTest.php

@@ -0,0 +1,183 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2021, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Test\src\Object\Log;
+
+use Friendica\Object\Log\ParsedLogLine;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Log parser testing class
+ */
+class ParsedLogLineTest extends TestCase
+{
+	public static function do_log_line($logline, $expected_data)
+	{
+		$parsed = new ParsedLogLine(0, $logline);
+		foreach ($expected_data as $k => $v) {
+			self::assertSame($parsed->$k, $v, '"'.$k.'" does not match expectation');
+		}
+	}
+
+	/**
+	 * test parsing a generic log line
+	 */
+	public function testGenericLogLine()
+	{
+		self::do_log_line(
+			'2021-05-24T15:40:01Z worker [WARNING]: Spool file does does not start with "item-" {"file":".","worker_id":"560c8b6","worker_cmd":"SpoolPost"} - {"file":"SpoolPost.php","line":40,"function":"execute","uid":"fd8c37","process_id":20846}',
+			[
+				'date'    => '2021-05-24T15:40:01Z',
+				'context' => 'worker',
+				'level'   => 'WARNING',
+				'message' => 'Spool file does does not start with "item-"',
+				'data'    => '{"file":".","worker_id":"560c8b6","worker_cmd":"SpoolPost"}',
+				'source'  => '{"file":"SpoolPost.php","line":40,"function":"execute","uid":"fd8c37","process_id":20846}',
+			]
+		);
+	}
+
+	/**
+	 * test parsing a log line with empty data
+	 */
+	public function testEmptyDataLogLine()
+	{
+		self::do_log_line(
+			'2021-05-24T15:23:58Z index [INFO]: No HTTP_SIGNATURE header [] - {"file":"HTTPSignature.php","line":476,"function":"getSigner","uid":"0a3934","process_id":14826}',
+			[
+				'date'    => '2021-05-24T15:23:58Z',
+				'context' => 'index',
+				'level'   => 'INFO',
+				'message' => 'No HTTP_SIGNATURE header',
+				'data'    => null,
+				'source'  => '{"file":"HTTPSignature.php","line":476,"function":"getSigner","uid":"0a3934","process_id":14826}',
+			]
+		);
+	}
+
+	/**
+	 * test parsing a log line with various " - " in it
+	 */
+	public function testTrickyDashLogLine()
+	{
+		self::do_log_line(
+			'2021-05-24T15:30:01Z worker [NOTICE]: Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10 {"worker_id":"ece8fc8","worker_cmd":"Cron"} - {"file":"Worker.php","line":786,"function":"tooMuchWorkers","uid":"364d3c","process_id":20754}',
+			[
+				'date'    => '2021-05-24T15:30:01Z',
+				'context' => 'worker',
+				'level'   => 'NOTICE',
+				'message' => 'Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10',
+				'data'    => '{"worker_id":"ece8fc8","worker_cmd":"Cron"}',
+				'source'  => '{"file":"Worker.php","line":786,"function":"tooMuchWorkers","uid":"364d3c","process_id":20754}',
+			]
+		);
+	}
+
+	/**
+	 * test non conforming log line
+	 */
+	public function testNonConformingLogLine()
+	{
+		self::do_log_line(
+			'this log line is not formatted as expected',
+			[
+				'date'    => null,
+				'context' => null,
+				'level'   => null,
+				'message' => 'this log line is not formatted as expected',
+				'data'    => null,
+				'source'  => null,
+			]
+		);
+	}
+
+	/**
+	 * test missing source
+	 */
+	public function testMissingSource()
+	{
+		self::do_log_line(
+			'2021-05-24T15:30:01Z worker [NOTICE]: Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10 {"worker_id":"ece8fc8","worker_cmd":"Cron"}',
+			[
+				'date'    => '2021-05-24T15:30:01Z',
+				'context' => 'worker',
+				'level'   => 'NOTICE',
+				'message' => 'Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10',
+				'data'    => '{"worker_id":"ece8fc8","worker_cmd":"Cron"}',
+				'source'  => null,
+			]
+		);
+	}
+
+	/**
+	 * test missing data
+	 */
+	public function testMissingData()
+	{
+		self::do_log_line(
+			'2021-05-24T15:30:01Z worker [NOTICE]: Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10 - {"file":"Worker.php","line":786,"function":"tooMuchWorkers","uid":"364d3c","process_id":20754}',
+			[
+				'date'    => '2021-05-24T15:30:01Z',
+				'context' => 'worker',
+				'level'   => 'NOTICE',
+				'message' => 'Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10',
+				'data'    => null,
+				'source'  => '{"file":"Worker.php","line":786,"function":"tooMuchWorkers","uid":"364d3c","process_id":20754}',
+			]
+		);
+	}
+
+	/**
+	 * test missing data and source
+	 */
+	public function testMissingDataAndSource()
+	{
+		self::do_log_line(
+			'2021-05-24T15:30:01Z worker [NOTICE]: Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10',
+			[
+				'date'    => '2021-05-24T15:30:01Z',
+				'context' => 'worker',
+				'level'   => 'NOTICE',
+				'message' => 'Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10',
+				'data'    => null,
+				'source'  => null,
+			]
+		);
+	}
+
+	/**
+	 * test missing source and invalid data
+	 */
+	public function testMissingSourceAndInvalidData()
+	{
+		self::do_log_line(
+			'2021-05-24T15:30:01Z worker [NOTICE]: Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10 {"invalidjson {really',
+			[
+				'date'    => '2021-05-24T15:30:01Z',
+				'context' => 'worker',
+				'level'   => 'NOTICE',
+				'message' => 'Load: 0.01/20 - processes: 0/1/6 (0:0, 30:1) - maximum: 10/10 {"invalidjson {really',
+				'data'    => null,
+				'source'  => null,
+			]
+		);
+	}
+}

view/js/module/admin/logs/view.js

@@ -0,0 +1,33 @@
+(function(){
+	function log_show_details(elm) {
+		const id = elm.id;
+		var hidden = true;
+		document
+			.querySelectorAll('[data-id="' + id + '"]')
+			.forEach(edetails => {
+				hidden = edetails.classList.toggle('hidden');
+			});
+		document
+			.querySelectorAll('[aria-expanded="true"]')
+			.forEach(eexpanded => {
+				eexpanded.setAttribute('aria-expanded', false);
+			});
+		
+		if (!hidden) {
+			elm.setAttribute('aria-expanded', true);
+		}
+	}
+
+	document
+		.querySelectorAll('.log-event')
+		.forEach(elm => {
+			elm.addEventListener("click", evt => {
+				log_show_details(evt.currentTarget);
+			});
+			elm.addEventListener("keydown", evt => {
+				if (evt.keyCode == 13 || evt.keyCode == 32) {
+					log_show_details(evt.currentTarget);
+				}
+			});
+		});
+})();

view/templates/admin/logs/view.tpl

@@ -1,6 +1,78 @@
 <div id="adminpage">
 	<h1>{{$title}} - {{$page}}</h1>
 
-	<h3>{{$logname}}</h3>
+	<h2>{{$logname}}</h2>
-	<div style="width:100%; height:400px; overflow: auto; "><pre>{{$data}}</pre></div>
+	{{if $error }}
+		<div id="admin-error-message-wrapper" class="alert alert-warning">
+			<p>{{$error nofilter}}</p>
+		</div>
+	{{else}}
+		<form>
+			<p>
+				<input type="search" name="q" value="{{$q}}" placeholder="search"></input>
+				<input type="Submit" value="search">
+				<a href="/admin/logs/view">clear</a>
+			</p>
+
+
+			<table>
+				<thead>
+					<tr>
+						<th>Date</th>
+						<th>
+							<select name="level" onchange="this.form.submit()">
+								{{foreach $filtersvalues.level as $v }}
+								<option {{if $filters.level == $v}}selected{{/if}} value="{{$v}}">
+									{{if $v == ""}}Level{{/if}}
+									{{$v}}
+								</option>
+								{{/foreach}}
+							</select>
+						</th>
+						<th>
+							<select name="context" onchange="this.form.submit()">
+								{{foreach $filtersvalues.context as $v }}
+								<option {{if $filters.context == $v}}selected{{/if}} value="{{$v}}">
+									{{if $v == ""}}Context{{/if}}
+									{{$v}}
+								</option>
+								{{/foreach}}
+							</select>
+						</th>
+						<th>Message</th>
+					</tr>
+				</thead>
+				<tbody>
+					{{foreach $data as $row}}
+						<tr id="ev-{{$row->id}}" class="log-event"
+						 role="button" tabIndex="0"
+						 aria-label="View details" aria-haspopup="true" aria-expanded="false"
+						 style="cursor:pointer;"
+						 title="Click to view details">
+							<td>{{$row->date}}</td>
+							<td>{{$row->level}}</td>
+							<td>{{$row->context}}</td>
+							<td>{{$row->message}}</td>
+						</tr>
+						<tr class="hidden" data-id="ev-{{$row->id}}"><th colspan="4">Data</th></tr>
+						{{foreach $row->getData() as $k=>$v}}
+							<tr class="hidden" data-id="ev-{{$row->id}}">
+								<th>{{$k}}</th>
+								<td colspan="3">
+									<pre>{{$v nofilter}}</pre>
+								</td>
+							</tr>
+						{{/foreach}}
+						<tr class="hidden" data-id="ev-{{$row->id}}"><th colspan="4">Source</th></tr>
+						{{foreach $row->getSource() as $k=>$v}}
+							<tr class="hidden" data-id="ev-{{$row->id}}">
+								<th>{{$k}}</th>
+								<td colspan="3">{{$v}}</td>
+							</tr>
+						{{/foreach}}
+					{{/foreach}}
+				</tbody>
+			</table>
+		</form>
+	{{/if}}
 </div>

view/theme/frio/css/style.css

@@ -84,6 +84,27 @@ blockquote {
 	overflow: hidden !important;
 }
 
+/**
+ * details tag
+ */
+details {
+	padding: .5em .5em 0;
+}
+details details {
+	padding-left: .5em;
+}
+details summary {
+	font-weight: bold;
+	display: list-item;
+}
+
+/**
+ * clickable table rows
+ */
+.table > tbody > td[role="button"] {
+	cursor: pointer;
+}
+
 /**
  * mobile aside
  */

view/theme/frio/js/module/admin/logs/view.js

@@ -0,0 +1,98 @@
+$(function(){
+
+	/* column filter */
+	$("a[data-filter]").on("click", function(ev) {
+		var filter = this.dataset.filter;
+		var value = this.dataset.filterValue;
+		var re = RegExp(filter+"=[a-z]*");
+		var newhref = location.href;
+		if (!location.href.indexOf("?") < 0) {
+			newhref = location.href + "?" + filter + "=" + value;
+		} else if (location.href.match(re)) {
+			newhref = location.href.replace(RegExp(filter+"=[a-z]*"), filter+"="+value);
+		} else {
+			newhref = location.href + "&" + filter + "=" + value;
+		}
+		location.href = newhref;
+		return false;
+	});
+
+	/* log details dialog */
+	$(".log-event").on("click", function(ev) {
+		show_details_for_element(ev.currentTarget);
+	});
+	$(".log-event").on("keydown", function(ev) {
+		if (ev.keyCode == 13 || ev.keyCode == 32) {
+			show_details_for_element(ev.currentTarget);
+		}
+	});
+
+
+	$("[data-previous").on("click", function(ev){ 
+		var currentid = document.getElementById("logdetail").dataset.rowId;
+		var $elm = $("#" + currentid).prev();
+		if ($elm.length == 0) return;
+		show_details_for_element($elm[0]);
+	});
+
+	$("[data-next").on("click", function(ev){ 
+		var currentid = document.getElementById("logdetail").dataset.rowId;
+		var $elm = $("#" + currentid).next();
+		if ($elm.length == 0) return;
+		show_details_for_element($elm[0]);
+	});
+
+
+	const $modal = $("#logdetail");
+
+	$modal.on("hidden.bs.modal", function(ev){
+		document
+			.querySelectorAll('[aria-expanded="true"]')
+			.forEach(elm => elm.setAttribute("aria-expanded", false))
+	});
+
+	function show_details_for_element(element) {
+		$modal[0].dataset.rowId = element.id;
+
+		var tr = $modal.find(".main-data tbody tr")[0];
+		tr.innerHTML = element.innerHTML;
+		
+		var data = JSON.parse(element.dataset.source);
+		$modal.find(".source-data td").each(function(i,elm){
+			var k = elm.dataset.value;
+			elm.innerText = data[k];
+		});
+
+		var elm = $modal.find(".event-data")[0];
+		elm.innerHTML = "";
+		var data = element.dataset.data;
+		if (data !== "") {
+			elm.innerHTML = "<h3>Data</h3>";
+			data = JSON.parse(data);
+			elm.innerHTML += recursive_details("", data);
+		}
+
+		$("[data-previous").prop("disabled", $(element).prev().length == 0);
+		$("[data-next").prop("disabled", $(element).next().length == 0);
+		
+		$modal.modal({})
+		element.setAttribute("aria-expanded", true);
+	}
+
+	function recursive_details(s, data, lev=0) {
+		for(var k in data) {
+			if (data.hasOwnProperty(k)) {
+				var v = data[k];
+				var open = lev > 1 ? "" : "open";
+				s += "<details " + open + "><summary>" + k + "</summary>";
+				if (typeof v === 'object' && v !== null) {
+					s = recursive_details(s, v, lev+1);
+				} else {
+					s +=  $("<pre>").text(v)[0].outerHTML;
+				}
+				s += "</details>";
+			}
+		}
+		return s;
+	}
+});

view/theme/frio/templates/admin/logs/view.tpl

@@ -0,0 +1,138 @@
+<div id="adminpage">
+	<h1>{{$title}} - {{$page}}</h1>
+
+	<h2>{{$logname}}</h2>
+	{{if $error }}
+		<div id="admin-error-message-wrapper" class="alert alert-warning">
+			<p>{{$error nofilter}}</p>
+		</div>
+	{{else}}
+		<form method="get" class="row">
+			<div class="col-xs-10">
+				<div class="form-group form-group-search">
+					<input accesskey="s" id="nav-search-input-field" class="form-control form-search"
+						type="text" name="q" data-toggle="tooltip" title="Search in logs"
+						placeholder="Search" value="{{$q}}">
+					<button class="btn btn-default btn-sm form-button-search"
+						type="submit">Search</button>
+				</div>
+			</div>
+			<div class="xol-xs-2">
+				<a href="/admin/logs/view" class="btn btn-default">Show all</a>
+			</div>
+		</form>
+
+		<table class="table table-hover">
+			<thead>
+				<tr>
+					<th>Date</th>
+					<th class="dropdown">
+						<a class="dropdown-toggle text-nowrap" type="button" id="level" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+							Level {{if $filters.level}}({{$filters.level}}){{/if}}<span class="caret"></span>
+						</a>
+						<ul class="dropdown-menu" aria-labelledby="level">
+							{{foreach $filtersvalues.level as $v }}
+								<li {{if $filters.level == $v}}class="active"{{/if}}>
+									<a href="/admin/logs/view?level={{$v}}" data-filter="level" data-filter-value="{{$v}}">
+										{{if $v == ""}}ALL{{/if}}{{$v}}
+									</a>
+								</li>
+							{{/foreach}}
+						</ul>
+					</th>
+					<th class="dropdown">
+						<a class="dropdown-toggle text-nowrap" type="button" id="context" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+							Context {{if $filters.context}}({{$filters.context}}){{/if}}<span class="caret"></span>
+						</a>
+						<ul class="dropdown-menu" aria-labelledby="context">
+							{{foreach $filtersvalues.context as $v }}
+								<li {{if $filters.context == $v}}class="active"{{/if}}>
+									<a href="/admin/logs/view?context={{$v}}" data-filter="context" data-filter-value="{{$v}}">
+										{{if $v == ""}}ALL{{/if}}{{$v}}
+									</a>
+								</li>
+							{{/foreach}}
+						</ul>
+					</th>
+					<th>Message</th>
+				</tr>
+			</thead>
+			<tbody>
+				{{foreach $data as $row}}
+				<tr id="ev-{{$row->id}}" class="log-event" 
+					role="button" tabIndex="0"
+					aria-label="View details" aria-haspopup="true" aria-expanded="false"
+					data-data="{{$row->data}}" data-source="{{$row->source}}">
+					<td>{{$row->date}}</td>
+					<td class="
+						{{if $row->level == "CRITICAL"}}bg-danger
+						{{elseif $row->level == "ERROR"}}bg-danger
+						{{elseif $row->level == "WARNING"}}bg-warinig
+						{{elseif $row->level == "NOTICE"}}bg-info
+						{{elseif $row->level == "DEBUG"}}text-muted
+						{{/if}}
+					">{{$row->level}}</td>
+					<td>{{$row->context}}</td>
+					<td style="width:80%">{{$row->message}}</td>
+				</tr>
+				{{/foreach}}
+			</tbody>
+		</table>
+	{{/if}}
+</div>
+
+<div id="logdetail" class="modal fade" tabindex="-1" role="dialog">
+	<div class="modal-dialog modal-lg" style="width:90%" role="document">
+		<div class="modal-content">
+			<div class="modal-header">
+				<button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+				<h4 class="modal-title">Event details</h4>
+			</div>
+			<div class="modal-body">
+				<table class="table main-data">
+					<thead>
+						<tr>
+							<th>Date</th>
+							<th>Level</th>
+							<th>Context</th>
+							<th>Message</th>
+						</tr>
+					</thead>
+					<tbody><tr></tr></tbody>
+				</table>
+				<table class="table source-data">
+					<thead>
+						<tr>
+							<th>File</th>
+							<th>Line</th>
+							<th>Function</th>
+							<th>UID</th>
+							<th>Process ID</th>
+						</tr>
+					</thead>
+					<tbody>
+						<tr>
+							<td data-value="file"></td>
+							<td data-value="line"></td>
+							<td data-value="function" style="width:70%"></td>
+							<td data-value="uid"></td>
+							<td data-value="process_id"></td>
+						</tr>
+					</tbody>
+				</table>
+
+
+				<div class="event-source">
+				</div>
+				<div class="event-data">
+				</div>
+
+			</div>
+			<div class="modal-footer">
+				<button type="button" class="btn btn-default" data-previous>&lt;</button>
+				<button type="button" class="btn btn-default" data-next>&gt;</button>
+				<button type="button" class="btn btn-primary" data-dismiss="modal">Close</button>
+			</div>
+		</div><!-- /.modal-content -->
+	</div><!-- /.modal-dialog -->
+</div><!-- /.modal -->