streams/Code/Module/Import.php
2024-01-02 15:47:27 +11:00

737 lines
28 KiB
PHP
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<?php
namespace Code\Module;
use App;
use URLify;
use Code\Web\Controller;
use Code\Web\HTTPSig;
use Code\Lib\Libzot;
use Code\Lib\Connect;
use Code\Lib\Channel;
use Code\Lib\Multibase;
use Code\Daemon\Run;
use Code\Import\Friendica;
use Code\Lib\ServiceClass;
use Code\Extend\Hook;
use Code\Render\Theme;
use Code\Lib\Url;
require_once('include/import.php');
require_once('include/photo_factory.php');
/**
* @brief Module for channel import.
*
* Import a channel, either by direct file upload or via
* connection to another server.
*/
class Import extends Controller
{
/**
* @brief Import channel into account.
*
* @param int $account_id
*/
public function import_account($account_id)
{
if (!$account_id) {
logger('No account ID supplied');
return;
}
$max_friends = ServiceClass::account_fetch($account_id, 'total_channels');
$max_feeds = ServiceClass::account_fetch($account_id, 'total_feeds');
$data = null;
$seize = ((x($_REQUEST, 'make_primary')) ? intval($_REQUEST['make_primary']) : 0);
$import_posts = 0;
$src = $_FILES['filename']['tmp_name'];
$filename = basename($_FILES['filename']['name']);
$filesize = intval($_FILES['filename']['size']);
$filetype = $_FILES['filename']['type'];
$newname = trim(strtolower($_REQUEST['newname']));
// import channel from file
if ($src) {
// This is OS specific and could also fail if your tmpdir isn't very
// large mostly used for Diaspora which exports gzipped files.
if (strpos($filename, '.gz')) {
@rename($src, $src . '.gz');
@system('gunzip ' . escapeshellarg($src . '.gz'));
}
if ($filesize) {
$data = @file_get_contents($src);
}
unlink($src);
}
else {
$import_posts = true;
}
// import channel from another server
if (!$src) {
$old_address = ((x($_REQUEST, 'old_address')) ? $_REQUEST['old_address'] : '');
if (!$old_address) {
logger('Nothing to import.');
notice(t('Nothing to import.') . EOL);
return;
} elseif (strpos($old_address, '')) {
// if you copy the identity address from your profile page, make it work for convenience
// - WARNING: this is a utf-8 variant and NOT an ASCII ampersand. Please do not edit.
$old_address = str_replace('', '@', $old_address);
}
$split = explode('@', $old_address);
if (count($split) !== 2) {
notice(t('An invalid channel address was entered. This resembles an email address (xyz@example.com).'));
return;
}
$channelname = $split[0];
$servername = $split[1];
$password = ((x($_REQUEST, 'old_password')) ? $_REQUEST['old_password'] : '');
$api_path = probe_api_path($servername);
if (!$api_path) {
notice(t('Unable to connect to old server') . EOL);
return;
}
$api_path .= 'channel/export/basic?f=&zap_compat=1';
$opts = ['http_auth' => $channelname . ':' . $password];
$ret = Url::get($api_path, $opts);
if ($ret['success']) {
$data = $ret['body'];
} else {
notice(t('Unable to download data from old server') . EOL);
return;
}
}
if (!$data) {
logger('Empty import file.');
notice(t('Imported file is empty.') . EOL);
return;
}
$data = json_decode($data, true);
//logger('import: data: ' . print_r($data,true));
//print_r($data);
// handle Friendica export
if (array_path_exists('user/parent-uid', $data)) {
$settings = ['account_id' => $account_id, 'seize' => 1, 'newname' => $newname];
$f = new Friendica($data, $settings);
return;
}
if (!array_key_exists('compatibility', $data)) {
Hook::call('import_foreign_channel_data', $data);
if ($data['handled']) {
return;
}
}
$codebase = 'zap';
if ((!array_path_exists('compatibility/codebase', $data)) || $data['compatibility']['codebase'] !== $codebase) {
notice('Data export format is not compatible with this software');
return;
}
$schema = (array_path_exists('compatibility/schema', $data) && $data['compatibility']['schema']) ? $data['compatibility']['schema'] : 'unknown';
// import channel
$relocate = ((array_key_exists('relocate', $data)) ? $data['relocate'] : null);
if (array_key_exists('channel', $data)) {
$max_identities = ServiceClass::account_fetch($account_id, 'total_identities');
if ($max_identities !== false) {
$r = q(
"select channel_id from channel where channel_account_id = %d and channel_removed = 0 ",
intval($account_id)
);
if ($r && count($r) > $max_identities) {
notice(sprintf(t('Your service plan only allows %d channels.'), $max_identities) . EOL);
return;
}
}
if ($newname) {
$x = false;
if (get_config('system', 'unicode_usernames')) {
$x = punify(mb_strtolower($newname));
}
if ((!$x) || strlen($x) > 64) {
$x = strtolower(URLify::transliterate($newname));
} else {
$x = $newname;
}
$newname = $x;
}
$channel = import_channel($data['channel'], $account_id, $seize, $newname);
}
else {
$channel = App::get_channel();
}
if (!$channel) {
logger('Channel not found. ' . print_r($channel, true));
notice(t('No channel. Import failed.') . EOL);
return;
}
if (is_array($data['config'])) {
import_config($channel, $data['config']);
}
logger('import step 2');
if (array_key_exists('channel', $data)) {
if (isset($data['photo']) && $data['photo']) {
import_channel_photo(base64url_decode($data['photo']['data']), $data['photo']['type'], $account_id, $channel['channel_id']);
}
if (is_array($data['profile'])) {
import_profiles($channel, $data['profile']);
}
}
logger('import step 3');
// import xchans and contact photos
// This *must* be done before importing hublocs
if (array_key_exists('channel', $data) && $seize) {
// replace any existing xchan we may have on this site if we're seizing control
$r = q(
"delete from xchan where xchan_hash = '%s'",
dbesc($channel['channel_hash'])
);
$r = xchan_store_lowlevel(
[
'xchan_hash' => $channel['channel_hash'],
'xchan_guid' => $channel['channel_guid'],
'xchan_guid_sig' => $channel['channel_guid_sig'],
'xchan_pubkey' => $channel['channel_pubkey'],
'xchan_epubkey' => (new Multibase())->publicKey($channel['channel_epubkey']),
'xchan_photo_l' => z_root() . "/photo/profile/l/" . $channel['channel_id'],
'xchan_photo_m' => z_root() . "/photo/profile/m/" . $channel['channel_id'],
'xchan_photo_s' => z_root() . "/photo/profile/s/" . $channel['channel_id'],
'xchan_photo_mimetype' => $data['photo']['type'],
'xchan_addr' => Channel::get_webfinger($channel),
'xchan_url' => z_root() . '/channel/' . $channel['channel_address'],
'xchan_connurl' => z_root() . '/poco/' . $channel['channel_address'],
'xchan_follow' => z_root() . '/follow?f=&url=%s',
'xchan_name' => $channel['channel_name'],
'xchan_network' => 'nomad',
'xchan_updated' => datetime_convert(),
'xchan_photo_date' => datetime_convert(),
'xchan_name_date' => datetime_convert()
]
);
}
logger('import step 4');
// import xchans
$xchans = $data['xchan'];
if ($xchans) {
foreach ($xchans as $xchan) {
// Provide backward compatibility for zot11 based projects
if ($xchan['xchan_network'] === 'nomad' && version_compare(NOMAD_PROTOCOL_VERSION, '10.0') <= 0) {
$xchan['xchan_network'] = 'zot6';
}
$hash = Libzot::make_xchan_hash($xchan['xchan_guid'], $xchan['xchan_pubkey']);
if (in_array($xchan['xchan_network'], ['nomad', 'zot6']) && $hash !== $xchan['xchan_hash']) {
logger('forged xchan: ' . print_r($xchan, true));
continue;
}
// xchan_pubforum was renamed but still may exist in Hubzilla imports
if (array_key_exists('xchan_pubforum', $xchan)) {
$xchan['xchan_type'] = $xchan['xchan_pubforum'];
unset($xchan['xchan_pubforum']);
}
$r = q(
"select xchan_hash from xchan where xchan_hash = '%s' limit 1",
dbesc($xchan['xchan_hash'])
);
if ($r) {
continue;
}
xchan_store_lowlevel($xchan);
if ($xchan['xchan_hash'] === $channel['channel_hash']) {
$r = q(
"update xchan set xchan_updated = '%s', xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s' where xchan_hash = '%s'",
dbesc(datetime_convert()),
dbesc(z_root() . '/photo/profile/l/' . $channel['channel_id']),
dbesc(z_root() . '/photo/profile/m/' . $channel['channel_id']),
dbesc(z_root() . '/photo/profile/s/' . $channel['channel_id']),
dbesc($xchan['xchan_hash'])
);
} else {
$photos = import_remote_xchan_photo($xchan['xchan_photo_l'], $xchan['xchan_hash']);
if ($photos) {
if ($photos[4]) {
$photodate = NULL_DATE;
} else {
$photodate = $xchan['xchan_photo_date'];
}
$r = q(
"update xchan set xchan_updated = '%s', xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s', xchan_photo_date = '%s' where xchan_hash = '%s'",
dbesc(datetime_convert()),
dbesc($photos[0]),
dbesc($photos[1]),
dbesc($photos[2]),
dbesc($photos[3]),
dbesc($photodate),
dbesc($xchan['xchan_hash'])
);
}
}
}
logger('import step 5');
}
logger('import step 6');
if (is_array($data['hubloc'])) {
import_hublocs($channel, $data['hubloc'], $seize);
}
logger('import step 7');
// create new hubloc for the new channel at this site
if (array_key_exists('channel', $data)) {
$r = hubloc_store_lowlevel(
[
'hubloc_guid' => $channel['channel_guid'],
'hubloc_guid_sig' => $channel['channel_guid_sig'],
'hubloc_id_url' => Channel::url($channel),
'hubloc_hash' => $channel['channel_hash'],
'hubloc_addr' => Channel::get_webfinger($channel),
'hubloc_network' => 'nomad',
'hubloc_primary' => (($seize) ? 1 : 0),
'hubloc_url' => z_root(),
'hubloc_url_sig' => Libzot::sign(z_root(), $channel['channel_prvkey']),
'hubloc_site_id' => Libzot::make_xchan_hash(z_root(), get_config('system', 'pubkey')),
'hubloc_host' => App::get_hostname(),
'hubloc_callback' => z_root() . '/nomad',
'hubloc_sitekey' => get_config('system', 'pubkey'),
'hubloc_updated' => datetime_convert()
]
);
// reset the original primary hubloc if it is being seized
if ($seize) {
$r = q(
"update hubloc set hubloc_primary = 0 where hubloc_primary = 1 and hubloc_hash = '%s' and hubloc_url != '%s' ",
dbesc($channel['channel_hash']),
dbesc(z_root())
);
}
}
$friends = 0;
$feeds = 0;
// import contacts
$abooks = $data['abook'];
if ($abooks) {
foreach ($abooks as $abook) {
$abook_copy = $abook;
$abconfig = null;
if (array_key_exists('abconfig', $abook) && is_array($abook['abconfig']) && count($abook['abconfig'])) {
$abconfig = $abook['abconfig'];
}
unset($abook['abook_id']);
unset($abook['abook_rating']);
unset($abook['abook_rating_text']);
unset($abook['abconfig']);
unset($abook['abook_their_perms']);
unset($abook['abook_my_perms']);
unset($abook['abook_not_here']);
$abook['abook_account'] = $account_id;
$abook['abook_channel'] = $channel['channel_id'];
$reconnect = false;
if (array_key_exists('abook_instance', $abook) && $abook['abook_instance'] && !str_contains($abook['abook_instance'], z_root())) {
$abook['abook_not_here'] = 1;
if (!($abook['abook_pending'] || $abook['abook_blocked'])) {
$reconnect = true;
}
}
if ($abook['abook_self']) {
$ctype = 0;
$role = get_pconfig($channel['channel_id'], 'system', 'permissions_role');
if (strpos($role, 'collection' !== false)) {
$ctype = 2;
} elseif (str_contains($role, 'group')) {
$ctype = 1;
}
if ($ctype) {
q(
"update xchan set xchan_type = %d where xchan_hash = '%s' ",
intval($ctype),
dbesc($abook['abook_xchan'])
);
}
} else {
if ($max_friends !== false && $friends > $max_friends) {
continue;
}
if ($max_feeds !== false && intval($abook['abook_feed']) && ($feeds > $max_feeds)) {
continue;
}
}
$r = q(
"select abook_id from abook where abook_xchan = '%s' and abook_channel = %d limit 1",
dbesc($abook['abook_xchan']),
intval($channel['channel_id'])
);
if ($r) {
$columns = db_columns('abook');
foreach ($abook as $k => $v) {
if (!in_array($k, $columns)) {
continue;
}
$r = q(
"UPDATE abook SET " . TQUOT . "%s" . TQUOT . " = '%s' WHERE abook_xchan = '%s' AND abook_channel = %d",
dbesc($k),
dbesc($v),
dbesc($abook['abook_xchan']),
intval($channel['channel_id'])
);
}
} else {
abook_store_lowlevel($abook);
$friends++;
if (intval($abook['abook_feed'])) {
$feeds++;
}
}
if ($abconfig) {
foreach ($abconfig as $abc) {
if ($abc['cat'] === 'system' && $abc['k'] === 'my_perms' && $schema !== 'streams') {
$x = explode(',', $abc['v']);
if (in_array('view_stream',$x) && ! in_array('deliver_stream',$x)) {
$x[] = 'deliver_stream';
}
set_abconfig($channel['channel_id'], $abc['xchan'], $abc['cat'], $abc['k'], implode(',', $x));
}
else {
set_abconfig($channel['channel_id'], $abc['xchan'], $abc['cat'], $abc['k'], $abc['v']);
}
}
}
if ($reconnect) {
Connect::connect($channel, $abook['abook_xchan']);
}
}
logger('import step 8');
}
// import groups
$groups = $data['group'];
if ($groups) {
$saved = [];
foreach ($groups as $group) {
$saved[$group['hash']] = ['old' => $group['id']];
if (array_key_exists('name', $group)) {
$group['gname'] = $group['name'];
unset($group['name']);
}
$r = q("select * from pgrp where gname = '%s' and uid = %d",
dbesc($group['gname']),
intval($channel['channel_id'])
);
if ($r) {
continue;
}
unset($group['id']);
$group['uid'] = $channel['channel_id'];
create_table_from_array('pgrp', $group);
}
// create a list of ids that applies to this system so we can map members to them
$r = q(
"select * from pgrp where uid = %d",
intval($channel['channel_id'])
);
if ($r) {
foreach ($r as $rr) {
$saved[$rr['hash']]['new'] = $rr['id'];
}
}
}
// import group members
$group_members = $data['group_member'];
if ($group_members) {
foreach ($group_members as $group_member) {
unset($group_member['id']);
$group_member['uid'] = $channel['channel_id'];
foreach ($saved as $x) {
if ($x['old'] == $group_member['gid']) {
$group_member['gid'] = $x['new'];
}
}
// check if it's a duplicate
$r = q("select * from pgrp_member where xchan = '%s' and gid = %d",
dbesc($group_member['xchan']),
intval($group_member['gid'])
);
if ($r) {
continue;
}
create_table_from_array('pgrp_member', $group_member);
}
}
logger('import step 9');
if (is_array($data['atoken'])) {
import_atoken($channel, $data['atoken']);
}
if (is_array($data['xign'])) {
import_xign($channel, $data['xign']);
}
if (is_array($data['block'])) {
import_block($channel, $data['block']);
}
if (is_array($data['block_xchan'])) {
import_xchans($data['block_xchan']);
}
if (is_array($data['obj'])) {
import_objs($channel, $data['obj']);
}
if (is_array($data['likes'])) {
import_likes($channel, $data['likes']);
}
if (is_array($data['app'])) {
import_apps($channel, $data['app']);
}
if (is_array($data['sysapp'])) {
import_sysapps($channel, $data['sysapp']);
}
if (is_array($data['chatroom'])) {
import_chatrooms($channel, $data['chatroom']);
}
// if (is_array($data['conv'])) {
// import_conv($channel,$data['conv']);
// }
// if (is_array($data['mail'])) {
// import_mail($channel,$data['mail']);
// }
if (is_array($data['event'])) {
import_events($channel, $data['event']);
}
if (is_array($data['event_item'])) {
import_items($channel, $data['event_item'], false, $relocate);
}
// if (is_array($data['menu'])) {
// import_menus($channel,$data['menu']);
// }
// if (is_array($data['wiki'])) {
// import_items($channel,$data['wiki'],false,$relocate);
// }
// if (is_array($data['webpages'])) {
// import_items($channel,$data['webpages'],false,$relocate);
// }
$addon = array('channel' => $channel, 'data' => $data);
Hook::call('import_channel', $addon);
$saved_notification_flags = Channel::notifications_off($channel['channel_id']);
if ($import_posts && array_key_exists('item', $data) && $data['item']) {
import_items($channel, $data['item'], false, $relocate);
}
if ($api_path && $import_posts) { // we are importing from a server and not a file
$m = parse_url($api_path);
$hz_server = $m['scheme'] . '://' . $m['host'];
$since = datetime_convert(date_default_timezone_get(), date_default_timezone_get(), '0001-01-01 00:00');
$until = datetime_convert(date_default_timezone_get(), date_default_timezone_get(), 'now + 1 day');
$poll_interval = get_config('system', 'poll_interval', 3);
$page = 0;
while (1) {
$headers = [
'X-API-Token' => random_string(),
'X-API-Request' => $hz_server . '/api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,
'Host' => $m['host'],
'(request-target)' => 'get /api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,
];
$headers = HTTPSig::create_sig($headers, $channel['channel_prvkey'], Channel::url($channel), true, 'sha512');
$x = Url::get($hz_server . '/api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page, ['headers' => $headers]);
if (!$x['success']) {
logger('no API response');
break;
}
$j = json_decode($x['body'], true);
if (!$j) {
break;
}
if (!(isset($j['item']) && is_array($j['item']) && count($j['item']))) {
break;
}
Run::Summon(['Content_importer', sprintf('%d', $page), $since, $until, $channel['channel_address'], urlencode($hz_server)]);
sleep($poll_interval);
$page++;
continue;
}
$headers = [
'X-API-Token' => random_string(),
'X-API-Request' => $hz_server . '/api/z/1.0/files?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until),
'Host' => $m['host'],
'(request-target)' => 'get /api/z/1.0/files?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until),
];
$headers = HTTPSig::create_sig($headers, $channel['channel_prvkey'], Channel::url($channel), true, 'sha512');
$x = Url::get($hz_server . '/api/z/1.0/files?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until), ['headers' => $headers]);
if (!$x['success']) {
logger('no API response');
return;
}
$j = json_decode($x['body'], true);
if (!$j) {
return;
}
if (!$j['success']) {
return;
}
$poll_interval = get_config('system', 'poll_interval', 3);
if (count($j['results'])) {
$todo = count($j['results']);
logger('total to process: ' . $todo, LOGGER_DEBUG);
foreach ($j['results'] as $jj) {
Run::Summon(['File_importer', $jj['hash'], $channel['channel_address'], urlencode($hz_server)]);
sleep($poll_interval);
}
}
notice(t('Files and Posts imported.') . EOL);
}
Channel::notifications_on($channel['channel_id'], $saved_notification_flags);
// send out refresh requests
// notify old server that it may no longer be primary.
Run::Summon(['Notifier', 'refresh_all', $channel['channel_id']]);
// This will indirectly perform a refresh_all *and* update the directory
Run::Summon(['Directory', $channel['channel_id']]);
notice(t('Import completed.') . EOL);
change_channel($channel['channel_id']);
goaway(z_root() . '/stream');
}
/**
* @brief Handle POST action on channel import page.
*/
public function post()
{
$account_id = get_account_id();
if (!$account_id) {
return;
}
check_form_security_token_redirectOnErr('/import', 'channel_import');
$this->import_account($account_id);
}
/**
* @brief Generate channel import page.
*
* @return string with parsed HTML.
*/
public function get()
{
if (!get_account_id()) {
notice(t('You must be logged in to use this feature.') . EOL);
return EMPTY_STR;
}
return replace_macros(Theme::get_template('channel_import.tpl'), [
'$title' => t('Import Channel'),
'$desc' => t('Use this form to make a copy (clone) of an existing channel from a different site/instance to this site. You may choose to retrieve the channel data from the old site via the network or provide an export file you downloaded previously.'),
'$label_filename' => t('File to upload'),
'$choice' => t('Or provide the connection details to an existing site/instance. This information is used immediately to download your existing content and is not saved or stored.'),
'$old_address' => ['old_address', t('Your old channel address (xyz@example.com)'), '', ''],
'$old_password' => ['old_password', t('Your old login password'), '', ''],
'$common' => t('Your primary location determines which URL or address should be displayed publicly. This should be the location you intend to use most often.'),
'$make_primary' => ['make_primary', t('Make this instance my primary location'), false, '', [t('No'), t('Yes')]],
'$newname' => ['newname', t('Use this channel nickname (optional)'), '', t('Leave blank to keep your existing channel nickname. You will be randomly assigned a similar nickname if either name is already allocated on this site.')],
'$pleasewait' => t('This process may take several minutes to complete and considerably longer if importing a large amount of posts and files. Please submit the form only once and leave this page open until finished.'),
'$form_security_token' => get_form_security_token('channel_import'),
'$submit' => t('Submit')
]);
}
}