Merge branch 'dev' of /home/macgirvin/z into dev

This commit is contained in:
nobody 2021-10-12 14:25:34 -07:00
commit 29ec3a5bab
12 changed files with 395 additions and 75 deletions

View file

@ -2177,6 +2177,10 @@ class Activity {
// we already have a stored record. Determine if it needs updating.
if ($ap_hubloc['hubloc_updated'] < datetime_convert('UTC','UTC',' now - ' . self::$ACTOR_CACHE_DAYS . ' days') || $force) {
$person_obj = self::fetch($url);
// ensure we received something
if (! is_array($person_obj)) {
return;
}
}
else {
return;

View file

@ -40,10 +40,10 @@ class JSalmon {
$ret = [ 'results' => [] ];
if(! is_array($x)) {
return $false;
return false;
}
if(! ( array_key_exists('signed',$x) && $x['signed'])) {
return $false;
return false;
}
$signed_data = preg_replace('/\s+/','',$x['data']) . '.'

View file

@ -1,7 +1,10 @@
<?php
namespace Zotlabs\Module;
// Connection autocompleter for util/nsh
// We could probably add this case to the general purpose autocompleter (mod_acl) but
// that module has gotten far too overloaded.
// Returns as json a simply array containing the webfinger addresses of all your Nomad connections
use Zotlabs\Web\Controller;
@ -16,6 +19,7 @@ class Connac extends Controller {
json_return_and_die($ret);
}
$r = q("select xchan_addr from abook left join xchan on abook_xchan = xchan_hash where abook_channel = %d and xchan_network = 'zot6'",
intval(local_channel())
);

View file

@ -56,7 +56,6 @@ class Photo extends Controller {
$channel = channelx_by_n($r[0]['uid']);
$obj = json_decode($r[0]['obj'],true);
$obj['actor'] = $obj['attributedTo'] = Activity::encode_person($channel,false);
as_return_and_die($obj,$channel);

View file

@ -18,7 +18,8 @@ class Stream extends Controller {
if (! local_channel()) {
return;
}
// setup identity information for page
$channel = App::get_channel();
App::$profile_uid = local_channel();
App::$data['channel'] = $channel;
@ -48,7 +49,7 @@ class Stream extends Controller {
$channel = ((isset(App::$data['channel'])) ? App::$data['channel'] : null);
// if called from liveUpdate() we will not have called Stream::init() on this request and $channel will not be set
// if called from liveUpdate() we will not have called Stream->init() on this request and $channel will not be set
if (! $channel) {
$channel = App::get_channel();

View file

@ -292,8 +292,9 @@ class HTTPSig {
// $force is used to ignore the local cache and only use the remote data; for instance the cached key might be stale
if (! $force) {
$x = q("select * from xchan left join hubloc on xchan_hash = hubloc_hash where hubloc_addr = '%s' or hubloc_id_url = '%s' order by hubloc_id desc",
$x = q("select * from xchan left join hubloc on xchan_hash = hubloc_hash where ( hubloc_addr = '%s' or hubloc_id_url = '%s' or hubloc_hash = '%s') order by hubloc_id desc",
dbesc(str_replace('acct:','',$cache_url)),
dbesc($cache_url),
dbesc($cache_url)
);

View file

@ -17,7 +17,7 @@ use Zotlabs\Daemon\Run;
* @brief This file defines some global constants and includes the central App class.
*/
define ( 'STD_VERSION', '21.10.02' );
define ( 'STD_VERSION', '21.10.06' );
define ( 'ZOT_REVISION', '10.0' );
define ( 'DB_UPDATE_VERSION', 1252 );

View file

@ -273,6 +273,7 @@ function photo_upload($channel, $observer, $args) {
$r0 = $ph->save($p);
$url[0] = [
'type' => 'Link',
'rel' => 'alternate',
'mediaType' => $type,
'summary' => $alt_desc,
'href' => z_root() . '/photo/' . $photo_hash . '-0.' . $ph->getExt(),
@ -294,6 +295,7 @@ function photo_upload($channel, $observer, $args) {
$r1 = $ph->storeThumbnail($p, PHOTO_RES_1024);
$url[1] = [
'type' => 'Link',
'rel' => 'alternate',
'mediaType' => $type,
'summary' => $alt_desc,
'href' => z_root() . '/photo/' . $photo_hash . '-1.' . $ph->getExt(),
@ -310,6 +312,7 @@ function photo_upload($channel, $observer, $args) {
$r2 = $ph->storeThumbnail($p, PHOTO_RES_640);
$url[2] = [
'type' => 'Link',
'rel' => 'alternate',
'mediaType' => $type,
'summary' => $alt_desc,
'href' => z_root() . '/photo/' . $photo_hash . '-2.' . $ph->getExt(),
@ -326,6 +329,7 @@ function photo_upload($channel, $observer, $args) {
$r3 = $ph->storeThumbnail($p, PHOTO_RES_320);
$url[3] = [
'type' => 'Link',
'rel' => 'alternate',
'mediaType' => $type,
'summary' => $alt_desc,
'href' => z_root() . '/photo/' . $photo_hash . '-3.' . $ph->getExt(),
@ -352,6 +356,7 @@ function photo_upload($channel, $observer, $args) {
$url[] = [
'type' => 'Link',
'rel' => 'about',
'mediaType' => 'text/html',
'href' => z_root() . '/photos/' . $channel['channel_address'] . '/image/' . $photo_hash
];
@ -403,6 +408,8 @@ function photo_upload($channel, $observer, $args) {
. $tag . z_root() . "/photo/{$photo_hash}-{$scale}." . $ph->getExt() . '[/zmg]'
. '[/zrl]';
$attribution = (($visitor) ? $visitor['xchan_url'] : $channel['xchan_url']);
// Create item object
$object = [
'type' => ACTIVITY_OBJ_PHOTO,
@ -410,6 +417,7 @@ function photo_upload($channel, $observer, $args) {
'summary' => $p['description'],
'published' => datetime_convert('UTC','UTC',$p['created'],ATOM_TIME),
'updated' => datetime_convert('UTC','UTC',$p['edited'],ATOM_TIME),
'attributedTo' => $attribution,
// This is a placeholder and will get over-ridden by the item mid, which is critical for sharing as a conversational item over activitypub
'id' => z_root() . '/photo/' . $photo_hash,
'url' => $url,

249
util/nsh
View file

@ -3,53 +3,121 @@
import sys, os
import readline
import pathlib
import urllib3
# use sys to amend the path so we can find/import the rest of our files
sys.path.append('util/py')
sys.path.append(str(pathlib.Path(__file__).parent.resolve()) + '/py')
import urllib
import configparser
import requests
import argparse
from requests.auth import HTTPBasicAuth
import easywebdav
import easywebdav.__version__ as easywebdavversion
import base64
__version__= "0.0.3"
# use sys and pathlib to amend the path so we can find/import easywebdav
sys.path.append('util/py')
sys.path.append(str(pathlib.Path(__file__).parent.resolve()) + '/py')
import easywebdav
import easywebdav.__version__ as easywebdavversion
__version__= "2021.10.06"
SERVER = None
USER = None
PASSWD = None
VERIFY_SSL=True
# this is filled in during the connection to your base server with a list of your Nomad connections
nomads = []
readline.parse_and_bind("tab:complete");
def complete(text,state):
class Completer():
basecmd = ['cd','ls','exists','mkdir','mkdirs','rmdir','delete','put','get', 'conn', 'connect', 'pwd','cat', 'lcd','lpwd', 'lls', 'quit', 'help']
matches = basecmd
matches = []
_commnd = []
_rfiles = []
_lfiles = []
_nomads = []
def __init__(self):
# Setup autocompletion
readline.parse_and_bind("tab:complete");
readline.set_completer(self.complete)
return
if get_cur_cmd() in [ 'conn', 'connect' ]:
matches = nomads
@property
def commnd(self):
return self._commnd
@commnd.setter
def commnd(self,names):
self._commnd = names
def get_command(self):
return self._commnd
@property
def lfiles(self):
return self._lfiles
@lfiles.setter
def lfiles(self,names):
self._lfiles = names
@property
def rfiles(self):
return self._rfiles
@rfiles.setter
def rfiles(self,names):
self._rfiles = names
@property
def nomads(self):
return self._nomads
@nomads.setter
def nomads(self,names):
self._nomads = names
# this is the completion function
def complete(self,text,state):
# set initial return state to base commands
self.matches = self._commnd
# peek at the typed input state and reconfigure the completion
# base map accordingly
current = self.get_cur_cmd()
if current in [ 'connect' ]:
self.matches = self._nomads
if current in [ 'put', 'lcd', 'lls' ]:
self.matches = self._lfiles
if current in [ 'get', 'cd', 'delete', 'cat', 'ls', 'rmdir', 'exists' ]:
self.matches = self._rfiles
if current in [ 'mkdir', 'mkdirs', 'pwd', 'lpwd', 'quit' ]:
self.matches = []
if text != "":
matches = [x for x in matches if x.startswith(text)]
# On the current "word" reduce the base map to a subset of the results
if text != "":
self.matches = [x + " " for x in self.matches if x.startswith(text)]
try:
response = self.matches[state]
except IndexError:
response = None
if matches == basecmd and state > len('connect'): #longest word in set
return None;
return matches[state]
return response
def get_cur_cmd(self):
idx = readline.get_begidx()
full = readline.get_line_buffer()
n = full[:idx].split()
return n[0] if len(n) > 0 else ""
readline.set_completer(complete)
def get_cur_cmd():
idx = readline.get_begidx()
full = readline.get_line_buffer()
n = full[:idx].split()
return n[0] if len(n) > 0 else ""
#####################################################
@ -59,9 +127,9 @@ class CommandNotFound(Exception):
class NSH(object):
commands = ['cd','ls','exists','mkdir','mkdirs','rmdir','delete','put','get',
'conn', 'connect', 'pwd','cat',
'connect', 'pwd','cat',
'lcd','lpwd', 'lls',
'quit', 'help']
'quit', 'help','rfiles']
def __init__(self, host, session=None, davclient=None):
self.sessions = {}
self.host = host
@ -70,7 +138,7 @@ class NSH(object):
@property
def host(self):
def host(self):
return self._host
@host.setter
@ -79,12 +147,12 @@ class NSH(object):
self._hostname = host.replace("https:","").replace("/","")
@property
def hostname(self):
def hostname(self):
return self._hostname
@hostname.setter
def hostname(self, hostname):
self._host = "https://%s/" % (hostname)
self._host = "https://{}/".format(hostname)
self._hostname = hostname
@property
@ -100,21 +168,20 @@ class NSH(object):
def PS1(self):
if self.davclient is None:
return "[!]> "
return "%s:%s> " % (self.hostname, self.davclient.cwd)
return "{}:{}> ".format(self.hostname, self.davclient.cwd)
def get_host_session(self, host=None):
if self.session is None:
session = requests.Session()
#session.params.update({'davguest':1})
else:
session = self.session
return session
def do(self, command, *args):
if not command in self.commands:
raise CommandNotFound("Unknown command '%s'" % command)
raise CommandNotFound("Unknown command '{}'".format(command))
cmd = getattr(self, "cmd_%s"%command, None)
cmd = getattr(self, "cmd_{}".format(command), None)
if cmd is None:
cmd = getattr(self.davclient, command)
@ -169,9 +236,6 @@ class NSH(object):
return self.davclient.download(args[0], args[1])
def cmd_conn(self, *args):
return self.do('connect', *args)
def cmd_connect(self, *args):
ruser = ''
if (len(args)==0):
@ -182,7 +246,7 @@ class NSH(object):
ruser = newhostname[0:i]
newhostname = newhostname[i+1:]
newhost = "https://%s/" % newhostname
newhost = "https://{}/".format(newhostname)
if newhostname == "~" or newhost == SERVER:
# back to home server
self.host = SERVER
@ -211,7 +275,7 @@ class NSH(object):
print('not found')
def cmd_pwd(self, *args):
return "%s%s" % ( self.davclient.baseurl, self.davclient.cwd )
return "{}:{}".format( self.hostname, self.davclient.cwd )
def cmd_ls(self, *args):
extra_args = ["-a", "-l", "-d"]
@ -225,9 +289,13 @@ class NSH(object):
r = self.davclient.ls(*args)
l = max([ len(str(f.size)) for f in r ] + [7,])
def _fmt(type, size, name):
def _fmt(typ, size, name):
clean = urllib.parse.unquote(name)
if clean != name:
name = name + " (\"" + clean.rstrip('/') + "\")"
if show_list:
return "%s %*d %s" % (type, l, f.size , name)
return "{t} {num: {width}} {nm}".format(t = typ, num = f.size, width = l, nm = name)
else:
return name
@ -246,6 +314,16 @@ class NSH(object):
if not show_only_dir or type=="d":
print( _fmt(type, f.size , name))
# This isn't a normal "user" command, but exists to update the
# autocompleter as it has intimate access to the current remote environment
def cmd_rfiles(self, *args):
ret = []
r = self.davclient.ls(*args)
for f in r:
ret.append(f.name.replace("/cloud" + self.davclient.cwd,""))
return ret
def cmd_lpwd(self, *args):
return os.getcwd()
@ -265,7 +343,8 @@ class NSH(object):
print()
print("Commands:")
for c in self.commands:
print("\t",c)
if c != 'rfiles':
print("\t",c)
print()
print("easywebdav", easywebdavversion.__version__, "(mod)")
print("requests", requests.__version__)
@ -277,18 +356,18 @@ class NSH(object):
resp = self.davclient._send('GET', rfile, (200,))
print(resp.text)
def load_conf():
def load_conf(conffile):
global SERVER,USER,PASSWD,VERIFY_SSL
homedir = os.getenv("HOME")
if homedir is None:
homedir = os.path.join(os.getenv("HOMEDRIVE"), os.getenv("HOMEPATH"))
optsfile = ".nshrc"
optsfile = ".nshrc" + "." + conffile if conffile else ".nshrc"
if not os.path.isfile(optsfile):
optsfile = os.path.join(homedir, ".nshrc")
optsfile = os.path.join(homedir, optsfile)
if not os.path.isfile(optsfile):
print("Please create a configuration file called '.nshrc':")
print("Please create a configuration file called '{}':".format(optsfile))
print("[nsh]")
print("host = https://yourhost.com/")
print("username = your_username")
@ -303,38 +382,57 @@ def load_conf():
if config.has_option('nsh', 'verify_ssl'):
VERIFY_SSL = config.getboolean('nsh', 'verify_ssl')
def get_lfiles():
ret = []
for f in os.listdir(os.getcwd()):
if os.path.isdir(f):
f=f+"/"
ret.append(f)
return ret
def nsh():
global nomads
nsh = NSH(SERVER)
completer = Completer()
session_home = nsh.get_host_session()
#~ #login on home server
print("logging in...")
if(sys.stdin.isatty()):
print("logging in...")
r = session_home.get(
SERVER + "/api/z/1.0/verify",
auth=HTTPBasicAuth(USER, PASSWD),
verify=VERIFY_SSL )
print("Hi - ", r.json()['channel_name'])
if(sys.stdin.isatty()):
print("Hi - ", r.json()['channel_name'])
nsh.session = session_home
r = session_home.get(SERVER + "/connac")
completer.commnd = ['cd','ls','exists','mkdir','mkdirs','rmdir','delete','put','get', 'connect', 'pwd','cat', 'lcd','lpwd', 'lls', 'quit', 'help']
# initialise list of available connections from json endpoint
# this will not change
r = session_home.get(SERVER + "/connac")
completer.nomads = r.json() if r else []
# initialise local file list
completer.lfiles = get_lfiles()
nomads = r.json()
# since the site directory may be empty, automatically cd to
# your own cloud storage folder
# your own cloud storage folder and update remote file list
nsh.do('cd', *[USER])
completer.rfiles = nsh.do('rfiles', *[])
# command loop
try:
input_str = input(nsh.PS1)
input_str = input(nsh.PS1 if sys.stdin.isatty() else "")
except EOFError as e:
input_str = "quit"
@ -348,13 +446,22 @@ def nsh():
args = toks[1:]
try:
ret = nsh.do(command, *args)
# update the internal file lists for the autocompleter if we just
# performed an action which may have changed them
if command in [ 'cd', 'connect', 'mkdir','mkdirs','rmdir','delete','put']:
completer.rfiles = nsh.do('rfiles', *[])
if command in [ 'get', 'lcd' ]:
completer.lfiles = get_lfiles()
except easywebdav.client.OperationFailed as e:
print(e)
except CommandNotFound as e:
print(e)
except urllib3.exceptions.NewConnectionError as e:
except urllib.exceptions.NewConnectionError as e:
print(e)
except urllib3.exceptions.MaxRetryError as e:
except urllib.exceptions.MaxRetryError as e:
print(e)
except requests.exceptions.ConnectionError as e:
print(e)
@ -363,14 +470,30 @@ def nsh():
print(ret)
try:
input_str = input(nsh.PS1)
input_str = input(nsh.PS1 if sys.stdin.isatty() else "")
except EOFError as e:
input_str = "quit"
if __name__=="__main__":
load_conf()
conffile = ""
parser = argparse.ArgumentParser(description = "Nomad shell - CLI for accessing local/remote Nomad|Zot cloud storage resources")
parser.add_argument('-c', nargs=1, metavar="name", help="load alternate configuration .nshrc.name", default="")
parser.add_argument('--version', action="store_true", help="print version and exit")
args = parser.parse_args()
if args.c:
conffile = args.c[0]
if args.version:
print (__version__)
sys.exit()
load_conf(conffile)
nsh()
sys.exit()

View file

@ -1,4 +1,4 @@
NSH - v.0.0.3
NSH - 2021.10.05
Client for browsing Nomad DAV repositories.
@ -18,7 +18,6 @@ Description
You can connect to a repository using
conn username@hostname
connect username@hostname
if you know a username on that site and if they have given you the requisite permission *or* their directory contains publicly readable content.
@ -35,13 +34,12 @@ to 'zotify' it. (See easywebdav/LICENSE)
Commands
--------
conn <hostname>
connect <hostname>
Authenticate to 'hostname' and switch to it. The root directory may be
hidden/empty. If it is, the only way to proceed is if you know a username on
that server. Then you can 'cd username'.
conn <username@hostname>
connect <username@hostname>
Authenticate to 'hostname' and switch to it and automatically cd to the 'username' directory
@ -112,6 +110,9 @@ to skip verification of ssl certs
Changelog
----------
2021.10.06 Add alternate configuration support and cmdline arg processing
2021.10.05 Add autocompletion
0.0.3 Convert to python3 and rename from zotsh to nsh
0.0.2 Fix "CommandNotFound" exception, new 'cat' command

54
util/py/jsalmon.py Normal file
View file

@ -0,0 +1,54 @@
from libzot import generate_rsa_keypair, rsa_sign, rsa_verify, base64urlnopad_encode, base64urlnopad_decode
import re
import json
class JSalmon:
def sign(data,key_id,key,data_type = 'application/x-zot+json'):
data = base64urlnopad_encode(data.encode("utf-8"))
encoding = 'base64url'
algorithm = 'RSA-SHA256'
data = re.sub(r'\s+',"",data)
fields = data + "." + base64urlnopad_encode(data_type.encode("utf-8")) + "." + base64urlnopad_encode(encoding.encode("utf-8")) + "." + base64urlnopad_encode(algorithm.encode("utf-8"))
signature = base64urlnopad_encode(rsa_sign(fields,key).encode("utf-8"))
return {
'signed' : True,
'data' : data,
'data_type' : data_type,
'encoding' : encoding,
'alg' : algorithm,
'sigs' : { 'value' : signature, 'key_id' : base64urlnopad_encode(key_id.encode("utf-8")) }}
def verify(x,key):
if x['signed'] != True:
return false
signed_data = re.sub(r'\s+','', x['data'] + "." + base64urlnopad_encode(x['data_type'].encode("utf-8")) + "." + base64urlnopad_encode(x['encoding'].encode("utf-8")) + "." + base64urlnopad_encode(x['alg'].encode("utf-8")))
binsig = base64urlnopad_decode(x['sigs']['value'])
if rsa_verify(signed_data,binsig,key) == True:
return True
return False
def unpack(data):
return json.loads(base64urlnopad_decode(data))
#if __name__=="__main__":
# prvkey,pubkey = generate_rsa_keypair()
# s = JSalmon.sign('abc123','mykeyid',prvkey)
# print (s)
# if JSalmon.verify(s,pubkey):
# print ('verified')
# else:
# print ('failed')

125
util/py/libzot.py Normal file
View file

@ -0,0 +1,125 @@
# libzot.py: crypto primitives to support Zot6/Nomad
import hashlib
import whirlpool
import base64
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa, padding
from cryptography.hazmat.primitives import serialization, hashes
from cryptography.exceptions import UnsupportedAlgorithm, AlreadyFinalized, InvalidSignature, NotYetFinalized, AlreadyUpdated, InvalidKey
# base64url implementations which support "no padding"
def base64urlnopad_encode(data: bytes) -> str:
return base64.urlsafe_b64encode(data).decode("utf-8").replace("=","");
def base64urlnopad_decode(data: str) -> bytes:
# restore any missing padding before calling the (strict) base64 decoder
if (data.find('=') == -1):
data += "=" * (-len(data) % 4)
return base64.urlsafe_b64decode(data)
def generate_rsa_keypair() -> (str, str):
key = rsa.generate_private_key(
public_exponent = 65537,
key_size = 4096,
backend = default_backend()
)
prvkey_pem = key.private_bytes(
encoding = serialization.Encoding.PEM,
format = serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm = serialization.NoEncryption()
)
pubkey = key.public_key()
pubkey_pem = pubkey.public_bytes(
encoding = serialization.Encoding.PEM,
format = serialization.PublicFormat.SubjectPublicKeyInfo,
)
# convert bytes to str
prvkey_pem = prvkey_pem.decode("utf-8")
pubkey_pem = pubkey_pem.decode("utf-8")
return prvkey_pem, pubkey_pem
def zot_sign(data: str, prvkey: str) -> str:
key = serialization.load_pem_private_key(prvkey.encode("ascii"),password = None)
rawsig = key.sign(hashlib.sha256(data.encode("utf-8")).hexdigest().encode("utf-8"), padding.PKCS1v15(), hashes.SHA256())
return 'sha256.' + base64.b64encode(rawsig).decode("utf-8")
def zot_verify(data: str, sig: str, pubkey: str) -> bool:
key = serialization.load_pem_public_key(pubkey.encode("ascii"))
alg, signature = sig.split('.')
if alg == 'sha256':
hashed = hashlib.sha256(data.encode("utf-8")).hexdigest().encode("utf-8")
algorithm = hashes.SHA256()
elif alg == 'sha512':
hashed = hashlib.sha256(data.encode("utf-8")).hexdigest().encode("utf-8")
algorithm = hashes.SHA512()
else:
hashed = ""
rawsig = base64.b64decode(signature)
try:
key.verify(rawsig, hashed, padding.PKCS1v15(), algorithm)
return True
except UnsupportedAlgorithm:
pass
except AlreadyFinalized:
pass
except InvalidSignature:
pass
except NotYetFinalized:
pass
except AlreadyUpdated:
pass
except InvalidKey:
pass
except BaseException:
pass
return False
def rsa_sign(data: str, prvkey: str) -> str:
key = serialization.load_pem_private_key(prvkey.encode("ascii"),password = None)
rawsig = key.sign(hashlib.sha256(data.encode("utf-8")).hexdigest().encode("utf-8"), padding.PKCS1v15(), hashes.SHA256())
return base64.b64encode(rawsig).decode("utf-8")
def rsa_verify(data: str, sig: str, pubkey: str) -> bool:
key = serialization.load_pem_public_key(pubkey.encode("ascii"))
hashed = hashlib.sha256(data.encode("utf-8")).hexdigest().encode("utf-8")
algorithm = hashes.SHA256()
rawsig = base64.b64decode(sig)
try:
key.verify(rawsig, hashed, padding.PKCS1v15(), algorithm)
return True
except UnsupportedAlgorithm:
pass
except AlreadyFinalized:
pass
except InvalidSignature:
pass
except NotYetFinalized:
pass
except AlreadyUpdated:
pass
except InvalidKey:
pass
except BaseException:
pass
return False
def make_xchan_hash(id_str: str,id_pubkey: str) -> str:
wp = whirlpool.new(id_str.encode("utf-8") + id_pubkey.encode("utf-8"))
return base64urlnopad_encode(wp.digest());