Friendica/friendica-github
1
0
Fork 0
mirror of https://github.com/friendica/friendica synced 2025-04-25 23:50:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

suppress some scraping errors when confronted with hybrid/strange

Browse source 
feeds that provide insufficient content-type and choke the html parser.
This commit is contained in:
Friendika 2011-02-02 14:48:27 -08:00
parent fce9988f73
commit ee45dee932
2 changed files with 21 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

20
include/Scrape.php
View file

@ -8,12 +8,18 @@ function scrape_dfrn($url) {
$a = get_app();
$ret = array();
logger('scrape_dfrn: url=' . $url);
$s = fetch_url($url);
if(! $s)
return $ret;
$headers = $a->get_curl_headers();
logger('scrape_dfrn: headers=' . $headers, LOGGER_DEBUG);
$lines = explode("\n",$headers);
if(count($lines)) {
foreach($lines as $line) {
@ -93,12 +99,17 @@ function scrape_meta($url) {
$a = get_app();
$ret = array();
logger('scrape_meta: url=' . $url);
$s = fetch_url($url);
if(! $s)
return $ret;
$headers = $a->get_curl_headers();
logger('scrape_meta: headers=' . $headers, LOGGER_DEBUG);
$lines = explode("\n",$headers);
if(count($lines)) {
foreach($lines as $line) {
@ -135,6 +146,9 @@ function scrape_vcard($url) {
$a = get_app();
$ret = array();
logger('scrape_vcard: url=' . $url);
$s = fetch_url($url);
if(! $s)
@ -190,15 +204,17 @@ function scrape_feed($url) {
return $ret;
$headers = $a->get_curl_headers();
logger('scrape_feed: headers=' . $headers, LOGGER_DEBUG);
$lines = explode("\n",$headers);
if(count($lines)) {
foreach($lines as $line) {
if(stristr($line,'content-type:')) {
if(stristr($line,'application/atom+xml')) {
if(stristr($line,'application/atom+xml') || stristr($s,'<feed')) {
$ret['feed_atom'] = $url;
return $ret;
}
if(stristr($line,'application/rss+xml')) {
if(stristr($line,'application/rss+xml') || stristr($s,'<rss')) {
$ret['feed_rss'] = $url;
return ret;
}