From 38fe3b8994167f8f3535e0d3eb067e43de35aba0 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 3 Jan 2020 09:09:03 -0500 Subject: [PATCH 1/2] Fix formatting in Feed::import --- src/Protocol/Feed.php | 49 +++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index ef90c40a2a..9ca4e7b3c2 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -14,8 +14,8 @@ use Friendica\Core\Protocol; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; -use Friendica\Util\ParseUrl; use Friendica\Util\Network; +use Friendica\Util\ParseUrl; use Friendica\Util\XML; /** @@ -35,15 +35,16 @@ class Feed { * @return array In simulation mode it returns the header and the first item * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function import($xml, $importer, &$contact, &$hub, $simulate = false) { - + public static function import($xml, $importer, &$contact, &$hub, $simulate = false) + { $a = \get_app(); if (!$simulate) { - Logger::log("Import Atom/RSS feed '".$contact["name"]."' (Contact ".$contact["id"].") for user ".$importer["uid"], Logger::DEBUG); + Logger::log("Import Atom/RSS feed '" . $contact["name"] . "' (Contact " . $contact["id"] . ") for user " . $importer["uid"], Logger::DEBUG); } else { Logger::log("Test Atom/RSS feed", Logger::DEBUG); } + if (empty($xml)) { Logger::log('XML is empty.', Logger::DEBUG); return; @@ -114,13 +115,16 @@ class Feed { if (empty($author["author-name"])) { $author["author-name"] = XML::getFirstNodeValue($xpath, '/atom:feed/atom:subtitle/text()'); } + if (empty($author["author-name"])) { $author["author-name"] = XML::getFirstNodeValue($xpath, '/atom:feed/atom:author/atom:name/text()'); } + $value = XML::getFirstNodeValue($xpath, 'atom:author/poco:displayName/text()'); if ($value != "") { $author["author-name"] = $value; } + if ($simulate) { $author["author-id"] = XML::getFirstNodeValue($xpath, '/atom:feed/atom:author/atom:id/text()'); @@ -134,14 +138,17 @@ class Feed { if ($value != "") { $author["author-nick"] = $value; } + $value = XML::getFirstNodeValue($xpath, 'atom:author/poco:address/poco:formatted/text()'); if ($value != "") { $author["author-location"] = $value; } + $value = XML::getFirstNodeValue($xpath, 'atom:author/poco:note/text()'); if ($value != "") { $author["author-about"] = $value; } + $avatar = XML::getFirstAttributes($xpath, "atom:author/atom:link[@rel='avatar']"); if (is_object($avatar)) { foreach ($avatar AS $attribute) { @@ -169,9 +176,11 @@ class Feed { if (empty($author["author-name"])) { $author["author-name"] = XML::getFirstNodeValue($xpath, '/rss/channel/copyright/text()'); } + if (empty($author["author-name"])) { $author["author-name"] = XML::getFirstNodeValue($xpath, '/rss/channel/description/text()'); } + $author["edited"] = $author["created"] = XML::getFirstNodeValue($xpath, '/rss/channel/pubDate/text()'); $author["app"] = XML::getFirstNodeValue($xpath, '/rss/channel/generator/text()'); @@ -185,6 +194,7 @@ class Feed { if (empty($author["author-name"])) { $author["author-name"] = $contact["name"]; } + $author["author-avatar"] = $contact["thumb"]; $author["owner-link"] = $contact["url"]; @@ -211,7 +221,7 @@ class Feed { $items = []; // Importing older entries first - for($i = $entries->length - 1; $i >= 0;--$i) { + for ($i = $entries->length - 1; $i >= 0; --$i) { $entry = $entries->item($i); $item = array_merge($header, $author); @@ -227,9 +237,11 @@ class Feed { } } } + if (empty($item["plink"])) { $item["plink"] = XML::getFirstNodeValue($xpath, 'link/text()', $entry); } + if (empty($item["plink"])) { $item["plink"] = XML::getFirstNodeValue($xpath, 'rss:link/text()', $entry); } @@ -239,6 +251,7 @@ class Feed { if (empty($item["uri"])) { $item["uri"] = XML::getFirstNodeValue($xpath, 'guid/text()', $entry); } + if (empty($item["uri"])) { $item["uri"] = $item["plink"]; } @@ -254,7 +267,7 @@ class Feed { $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN]; $previous = Item::selectFirst(['id'], $condition); if (DBA::isResult($previous)) { - Logger::log("Item with uri ".$item["uri"]." for user ".$importer["uid"]." already existed under id ".$previous["id"], Logger::DEBUG); + Logger::log("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"], Logger::DEBUG); continue; } } @@ -275,9 +288,11 @@ class Feed { if (empty($published)) { $published = XML::getFirstNodeValue($xpath, 'pubDate/text()', $entry); } + if (empty($published)) { $published = XML::getFirstNodeValue($xpath, 'dc:date/text()', $entry); } + $updated = XML::getFirstNodeValue($xpath, 'atom:updated/text()', $entry); if (empty($updated) && !empty($published)) { @@ -291,20 +306,25 @@ class Feed { if ($published != "") { $item["created"] = $published; } + if ($updated != "") { $item["edited"] = $updated; } + $creator = XML::getFirstNodeValue($xpath, 'author/text()', $entry); if (empty($creator)) { $creator = XML::getFirstNodeValue($xpath, 'atom:author/atom:name/text()', $entry); } + if (empty($creator)) { $creator = XML::getFirstNodeValue($xpath, 'dc:creator/text()', $entry); } + if ($creator != "") { $item["author-name"] = $creator; } + $creator = XML::getFirstNodeValue($xpath, 'dc:creator/text()', $entry); if ($creator != "") { @@ -332,6 +352,7 @@ class Feed { $type = $attribute->textContent; } } + if (!empty($item["attach"])) { $item["attach"] .= ','; } else { @@ -340,7 +361,7 @@ class Feed { $attachments[] = ["link" => $href, "type" => $type, "length" => $length]; - $item["attach"] .= '[attach]href="'.$href.'" length="'.$length.'" type="'.$type.'"[/attach]'; + $item["attach"] .= '[attach]href="' . $href . '" length="' . $length . '" type="' . $type . '"[/attach]'; } $tags = ''; @@ -418,7 +439,7 @@ class Feed { $item["body"] = trim($item["title"]); } - $data = ParseUrl::getSiteinfoCached($item['plink'], true); + $data = ParseUrl::getSiteinfoCached($item['plink'], true); if (!empty($data['text']) && !empty($data['title']) && (mb_strlen($item['body']) < mb_strlen($data['text']))) { // When the fetched page info text is longer than the body, we do try to enhance the body if (!empty($item['body']) && (strpos($data['title'], $item['body']) === false) && (strpos($data['text'], $item['body']) === false)) { @@ -432,7 +453,7 @@ class Feed { // We always strip the title since it will be added in the page information $item["title"] = ""; - $item["body"] = $item["body"].add_page_info($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); + $item["body"] = $item["body"] . add_page_info($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); $item["tag"] = add_page_keywords($item["plink"], $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]); $item["object-type"] = Activity\ObjectType::BOOKMARK; unset($item["attach"]); @@ -448,16 +469,17 @@ class Feed { // @todo $preview is never set in this case, is it intended? - @MrPetovan 2018-02-13 $item["tag"] = add_page_keywords($item["plink"], $preview, true, $contact["ffi_keyword_blacklist"]); } - $item["body"] .= "\n".$item['tag']; + $item["body"] .= "\n" . $item['tag']; } + // Add the link to the original feed entry if not present in feed if (($item['plink'] != '') && !strstr($item["body"], $item['plink'])) { - $item["body"] .= "[hr][url]".$item['plink']."[/url]"; + $item["body"] .= "[hr][url]" . $item['plink'] . "[/url]"; } } if (!$simulate) { - Logger::log("Stored feed: ".print_r($item, true), Logger::DEBUG); + Logger::log("Stored feed: " . print_r($item, true), Logger::DEBUG); $notify = Item::isRemoteSelf($contact, $item); @@ -474,10 +496,11 @@ class Feed { $id = Item::insert($item, false, $notify); - Logger::log("Feed for contact ".$contact["url"]." stored under id ".$id); + Logger::log("Feed for contact " . $contact["url"] . " stored under id " . $id); } else { $items[] = $item; } + if ($simulate) { break; } From ebaf7267440699fd5d34e7399e9d0ff754222e0a Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 3 Jan 2020 09:26:28 -0500 Subject: [PATCH 2/2] Rework Feed::import parameters - Remove unused $hub - Replace $simulate with an implicit condition --- include/items.php | 2 +- src/Module/Debug/Feed.php | 5 +--- src/Network/Probe.php | 8 ++---- src/Protocol/Feed.php | 53 +++++++++++++++++---------------------- 4 files changed, 27 insertions(+), 41 deletions(-) diff --git a/include/items.php b/include/items.php index d1f0ba8787..91165344af 100644 --- a/include/items.php +++ b/include/items.php @@ -258,7 +258,7 @@ function consume_feed($xml, array $importer, array $contact, &$hub) if ($contact['network'] === Protocol::FEED) { Logger::log("Consume feeds", Logger::DEBUG); - Feed::import($xml, $importer, $contact, $hub); + Feed::import($xml, $importer, $contact); return; } diff --git a/src/Module/Debug/Feed.php b/src/Module/Debug/Feed.php index 9a48d743c8..f3e8af0c57 100644 --- a/src/Module/Debug/Feed.php +++ b/src/Module/Debug/Feed.php @@ -29,15 +29,12 @@ class Feed extends BaseModule if (!empty($_REQUEST['url'])) { $url = $_REQUEST['url']; - $importer = Model\User::getById(local_user()); - $contact_id = Model\Contact::getIdForURL($url, local_user(), true); $contact = Model\Contact::getById($contact_id); $xml = Network::fetchUrl($contact['poll']); - $dummy = null; - $import_result = Protocol\Feed::import($xml, $importer, $contact, $dummy, true); + $import_result = Protocol\Feed::import($xml); $result = [ 'input' => $xml, diff --git a/src/Network/Probe.php b/src/Network/Probe.php index dac0822a94..122df70ce7 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -1480,10 +1480,7 @@ class Probe return false; } $feed = $curlResult->getBody(); - $dummy1 = null; - $dummy2 = null; - $dummy2 = null; - $feed_data = Feed::import($feed, $dummy1, $dummy2, $dummy3, true); + $feed_data = Feed::import($feed); if (!$feed_data) { return false; } @@ -1763,8 +1760,7 @@ class Probe return false; } $feed = $curlResult->getBody(); - $dummy1 = $dummy2 = $dummy3 = null; - $feed_data = Feed::import($feed, $dummy1, $dummy2, $dummy3, true); + $feed_data = Feed::import($feed); if (!$feed_data) { if (!$probe) { diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 9ca4e7b3c2..751a0f1e90 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -29,25 +29,23 @@ class Feed { * @param string $xml The feed data * @param array $importer The user record of the importer * @param array $contact The contact record of the feed - * @param string $hub Unused dummy value for compatibility reasons - * @param bool $simulate If enabled, no data is imported * - * @return array In simulation mode it returns the header and the first item + * @return array Returns the header and the first item in dry run mode * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function import($xml, $importer, &$contact, &$hub, $simulate = false) + public static function import($xml, array $importer = [], array $contact = []) { - $a = \get_app(); + $dryRun = empty($importer) && empty($contact); - if (!$simulate) { - Logger::log("Import Atom/RSS feed '" . $contact["name"] . "' (Contact " . $contact["id"] . ") for user " . $importer["uid"], Logger::DEBUG); + if ($dryRun) { + Logger::info("Test Atom/RSS feed"); } else { - Logger::log("Test Atom/RSS feed", Logger::DEBUG); + Logger::info("Import Atom/RSS feed '" . $contact["name"] . "' (Contact " . $contact["id"] . ") for user " . $importer["uid"]); } if (empty($xml)) { - Logger::log('XML is empty.', Logger::DEBUG); - return; + Logger::info('XML is empty.'); + return []; } if (!empty($contact['poll'])) { @@ -125,7 +123,7 @@ class Feed { $author["author-name"] = $value; } - if ($simulate) { + if ($dryRun) { $author["author-id"] = XML::getFirstNodeValue($xpath, '/atom:feed/atom:author/atom:id/text()'); // See https://tools.ietf.org/html/rfc4287#section-3.2.2 @@ -188,7 +186,7 @@ class Feed { $entries = $xpath->query('/rss/channel/item'); } - if (!$simulate) { + if (!$dryRun) { $author["author-link"] = $contact["url"]; if (empty($author["author-name"])) { @@ -203,7 +201,7 @@ class Feed { } $header = []; - $header["uid"] = $importer["uid"]; + $header["uid"] = $importer["uid"] ?? 0; $header["network"] = Protocol::FEED; $header["wall"] = 0; $header["origin"] = 0; @@ -212,11 +210,11 @@ class Feed { $header["verb"] = Activity::POST; $header["object-type"] = Activity\ObjectType::NOTE; - $header["contact-id"] = $contact["id"]; + $header["contact-id"] = $contact["id"] ?? 0; if (!is_object($entries)) { - Logger::log("There are no entries in this feed.", Logger::DEBUG); - return; + Logger::info("There are no entries in this feed."); + return []; } $items = []; @@ -262,12 +260,12 @@ class Feed { $item["parent-uri"] = $item["uri"]; - if (!$simulate) { + if (!$dryRun) { $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)", $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN]; $previous = Item::selectFirst(['id'], $condition); if (DBA::isResult($previous)) { - Logger::log("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"], Logger::DEBUG); + Logger::info("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"]); continue; } } @@ -478,8 +476,11 @@ class Feed { } } - if (!$simulate) { - Logger::log("Stored feed: " . print_r($item, true), Logger::DEBUG); + if ($dryRun) { + $items[] = $item; + break; + } else { + Logger::info("Stored feed: " . print_r($item, true)); $notify = Item::isRemoteSelf($contact, $item); @@ -496,19 +497,11 @@ class Feed { $id = Item::insert($item, false, $notify); - Logger::log("Feed for contact " . $contact["url"] . " stored under id " . $id); - } else { - $items[] = $item; - } - - if ($simulate) { - break; + Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); } } - if ($simulate) { - return ["header" => $author, "items" => $items]; - } + return ["header" => $author, "items" => $items]; } private static function titleIsBody($title, $body)