Merge pull request #1244 from pi-hole/new/FTL-binary

FTL

.gitignore

@@ -4,4 +4,3 @@
 __pycache__
 .cache
 .pullapprove.yml
-

advanced/Scripts/chronometer.sh

@@ -49,6 +49,9 @@ normalChrono() {
 		#uptime -p	#Doesn't work on all versions of uptime
 		uptime | awk -F'( |,|:)+' '{if ($7=="min") m=$6; else {if ($7~/^day/) {d=$6;h=$8;m=$9} else {h=$6;m=$7}}} {print d+0,"days,",h+0,"hours,",m+0,"minutes."}'
 		echo "-------------------------------"
+		domain=$(curl -s -X GET http://127.0.0.1/admin/api.php?recentBlocked)
+		echo "Recently blocked:"
+		echo "  $domain"
 		# Uncomment to continually read the log file and display the current domain being blocked
 		#tail -f /var/log/pihole.log | awk '/\/etc\/pihole\/gravity.list/ {if ($7 != "address" && $7 != "name" && $7 != "/etc/pihole/gravity.list") print $7; else;}'
 

advanced/Scripts/piholeDebug.sh

@@ -313,8 +313,8 @@ testResolver() {
 checkProcesses() {
 	header_write "Processes Check"
 
-	echo ":::     Logging status of lighttpd and dnsmasq..."
+	echo ":::     Logging status of lighttpd, dnsmasq and pihole-FTL..."
-	PROCESSES=( lighttpd dnsmasq )
+	PROCESSES=( lighttpd dnsmasq pihole-FTL )
 	for i in "${PROCESSES[@]}"; do
 		log_write ""
 		log_write "${i}"
@@ -363,6 +363,7 @@ ip_check 4 ${IPV4_ADDRESS}
 
 daemon_check lighttpd http
 daemon_check dnsmasq domain
+daemon_check pihole-FTL 4711
 checkProcesses
 testResolver
 debugLighttpd

advanced/Scripts/update.sh

@@ -75,6 +75,18 @@ GitCheckUpdateAvail() {
   fi
 }
 
+FTLcheckUpdate() {
+
+	local FTLversion=$(/usr/bin/pihole-FTL tag)
+	local FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n')
+
+	if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 main() {
   local pihole_version_current
   local web_version_current
@@ -96,6 +108,21 @@ main() {
     echo "::: Pi-hole Core:   up to date"
   fi
 
+  if FTLcheckUpdate ; then
+    FTL_update=true
+    echo "::: FTL:            update available"
+  else
+    FTL_update=false
+    echo "::: FTL:            up to date"
+  fi
+
+  if ${FTL_update}; then
+    echo ":::"
+    echo "::: FTL out of date"
+    FTLdetect
+    echo ":::"
+  fi
+
   if [[ ${INSTALL_WEB} == true ]]; then
     if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then
       echo "::: Critical Error: Web Admin repo is missing from system!"
@@ -122,9 +149,11 @@ main() {
     #            pull pihole repo run install --unattended
 
     if ! ${core_update} && ! ${web_update} ; then
+      if ! ${FTL_update} ; then
         echo ":::"
         echo "::: Everything is up to date!"
         exit 0
+      fi
 
     elif ! ${core_update} && ${web_update} ; then
       echo ":::"
@@ -139,7 +168,7 @@ main() {
 
     elif ${core_update} && ${web_update} ; then
       echo ":::"
-      echo "::: Updating Everything"
+      echo "::: Updating Pi-hole core and web admin files"
       getGitFiles "${PI_HOLE_FILES_DIR}" "${PI_HOLE_GIT_URL}"
       ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --unattended || echo "Unable to complete update, contact Pi-hole" && exit 1
     else
@@ -148,9 +177,11 @@ main() {
     fi
   else # Web Admin not installed, so only verify if core is up to date
     if ! ${core_update}; then
+      if ! ${FTL_update} ; then
         echo ":::"
         echo "::: Everything is up to date!"
         exit 0
+      fi
     else
       echo ":::"
       echo "::: Pi-hole core files out of date"
@@ -173,6 +204,15 @@ main() {
     echo "::: If you had made any changes in '/etc/.pihole/', they have been stashed using 'git stash'"
   fi
 
+  if [[ ${FTL_update} == true ]]; then
+    FTL_version_current="$(/usr/bin/pihole-FTL tag)"
+    echo ":::"
+    echo "::: FTL version is now at ${FTL_version_current}"
+    start_service pihole-FTL
+    enable_service pihole-FTL
+  fi
+
+
   echo ""
   exit 0
 

advanced/pihole-FTL.service

@@ -0,0 +1,82 @@
+#!/bin/bash
+### BEGIN INIT INFO
+# Provides:          pihole-FTL
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: pihole-FTL daemon
+# Description:       Enable service provided by pihole-FTL daemon
+### END INIT INFO
+
+FTLUSER=pihole
+PIDFILE=/var/run/pihole-FTL.pid
+
+get_pid() {
+    cat "$PIDFILE"
+}
+
+is_running() {
+    [ -f "$PIDFILE" ] && ps $(get_pid) > /dev/null 2>&1
+}
+
+# Start the service
+start() {
+  if is_running; then
+    echo "pihole-FTL is already running"
+  else
+    touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
+    chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
+    chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
+    su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER"
+    echo
+  fi
+}
+
+# Stop the service
+stop() {
+  if is_running; then
+    kill $(get_pid)
+    for i in {1..10}; do
+      if ! is_running; then
+        break
+      fi
+
+      echo -n "."
+      sleep 1
+    done
+    echo
+
+    if is_running; then
+      echo "Not stopped; may still be shutting down or shutdown may have failed"
+      exit 1
+    else
+      echo "Stopped"
+    fi
+  else
+    echo "Not running"
+  fi
+  echo
+}
+
+### main logic ###
+case "$1" in
+  start)
+        start
+        ;;
+  stop)
+        stop
+        ;;
+  status)
+        status pihole-FTL
+        ;;
+  restart|reload|condrestart)
+        stop
+        start
+        ;;
+  *)
+        echo $"Usage: $0 {start|stop|restart|reload|status}"
+        exit 1
+esac
+
+exit 0

automated install/basic-install.sh

@@ -1043,6 +1043,7 @@ installPihole() {
   fi
   installCron
   installLogrotate
+  FTLdetect || echo "::: FTL Engine not installed."
   configureFirewall
   finalExports
   #runGravity
@@ -1074,6 +1075,7 @@ updatePihole() {
   fi
   installCron
   installLogrotate
+  FTLdetect || echo "::: FTL Engine not installed."
   finalExports #re-export setupVars.conf to account for any new vars added in new versions
   #runGravity
 }
@@ -1167,6 +1169,93 @@ if [[ "${reconfigure}" == true ]]; then
     fi
 }
 
+FTLinstall() {
+  # Download and install FTL binary
+  local binary="${1}"
+  local latesttag
+  local orig_dir
+  echo -n ":::  Installing FTL... "
+
+  orig_dir="${PWD}"
+  latesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep "Location" | awk -F '/' '{print $NF}')
+  # Tags should always start with v, check for that.
+  if [[ ! "${latesttag}" == v* ]]; then
+    echo "failed (error in getting latest release location from GitHub)"
+    return 1
+  fi
+  if curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}/${binary}" -o "/tmp/${binary}"; then
+    # Get sha1 of the binary we just downloaded for verification.
+    curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}/${binary}.sha1" -o "/tmp/${binary}.sha1"
+    # Check if we just downloaded text, or a binary file.
+    cd /tmp
+    if sha1sum --status --quiet -c "${binary}".sha1; then
+      echo -n "transferred... "
+      stop_service pihole-FTL &> /dev/null
+      install -T -m 0755 /tmp/${binary} /usr/bin/pihole-FTL
+      cd "${orig_dir}"
+      install -T -m 0755 "/etc/.pihole/advanced/pihole-FTL.service" "/etc/init.d/pihole-FTL"
+      echo "done."
+      return 0
+    else
+      echo "failed (download of binary from Github failed)"
+      cd "${orig_dir}"
+      return 1
+    fi
+  else
+    cd "${orig_dir}"
+    echo "failed (URL not found.)"
+  fi
+}
+
+FTLdetect() {
+  # Detect suitable FTL binary platform
+  echo ":::"
+  echo "::: Downloading latest version of FTL..."
+
+  local machine
+  local binary
+
+  machine=$(uname -m)
+
+  if [[ $machine == arm* || $machine == *aarch* ]]; then
+    # ARM
+    local rev=$(uname -m | sed "s/[^0-9]//g;")
+    local lib=$(ldd /bin/ls | grep -E '^\s*/lib' | awk '{ print $1 }')
+    if [[ "$lib" == "/lib/ld-linux-aarch64.so.1" ]]; then
+      echo ":::  Detected ARM-aarch64 architecture"
+      binary="pihole-FTL-aarch64-linux-gnu"
+    elif [[ "$lib" == "/lib/ld-linux-armhf.so.3" ]]; then
+      if [ "$rev" -gt "6" ]; then
+        echo ":::  Detected ARM-hf architecture (armv7+)"
+        binary="pihole-FTL-arm-linux-gnueabihf"
+      else
+        echo ":::  Detected ARM-hf architecture (armv6 or lower)"
+        echo ":::  Using ARM binary"
+        binary="pihole-FTL-arm-linux-gnueabi"
+      fi
+    else
+      echo ":::  Detected ARM architecture"
+      binary="pihole-FTL-arm-linux-gnueabi"
+    fi
+  elif [[ $machine == x86_64 ]]; then
+    # 64bit
+    echo ":::  Detected x86_64 architecture"
+    binary="pihole-FTL-linux-x86_64"
+  else
+    # Something else - we try to use 32bit executable and warn the user
+    if [[ ! $machine == i686 ]]; then
+      echo ":::  Not able to detect architecture (unknown: ${machine}), trying 32bit executable"
+      echo ":::  Contact Pi-hole support if you experience problems (like FTL not running)"
+    else
+      echo ":::  Detected 32bit (i686) architecture"
+    fi
+    binary="pihole-FTL-linux-x86_32"
+  fi
+
+  FTLinstall "${binary}" || return 1
+
+}
+
 main() {
 
   ######## FIRST CHECK ########
@@ -1311,6 +1400,9 @@ main() {
 
   runGravity
 
+  start_service pihole-FTL
+  enable_service pihole-FTL
+
   echo "::: done."
 
   if [[ "${useUpdateVars}" == false ]]; then

test/debian.Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:jessie
+FROM buildpack-deps:jessie-scm
 
 ENV GITDIR /etc/.pihole
 ENV SCRIPTDIR /opt/pihole

test/test_automated_install.py

@@ -297,6 +297,111 @@ def test_update_package_cache_failure_no_errors(Pihole):
     assert 'ERROR' in updateCache.stdout
     assert 'done!' not in updateCache.stdout
 
+def test_FTL_detect_aarch64_no_errors(Pihole):
+    ''' confirms only aarch64 package is downloaded for FTL engine '''
+    # mock uname to return aarch64 platform
+    mock_command('uname', {'-m':('aarch64', '0')}, Pihole)
+    # mock ldd to respond with aarch64 shared library
+    mock_command('ldd', {'/bin/ls':('/lib/ld-linux-aarch64.so.1', '0')}, Pihole)
+    detectPlatform = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLdetect
+    ''')
+    expected_stdout = 'Detected ARM-aarch64 architecture'
+    assert expected_stdout in detectPlatform.stdout
+
+def test_FTL_detect_armv6l_no_errors(Pihole):
+    ''' confirms only armv6l package is downloaded for FTL engine '''
+    # mock uname to return armv6l platform
+    mock_command('uname', {'-m':('armv6l', '0')}, Pihole)
+    # mock ldd to respond with aarch64 shared library
+    mock_command('ldd', {'/bin/ls':('/lib/ld-linux-armhf.so.3', '0')}, Pihole)
+    detectPlatform = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLdetect
+    ''')
+    expected_stdout = 'Detected ARM-hf architecture (armv6 or lower)'
+    assert expected_stdout in detectPlatform.stdout
+
+def test_FTL_detect_armv7l_no_errors(Pihole):
+    ''' confirms only armv7l package is downloaded for FTL engine '''
+    # mock uname to return armv7l platform
+    mock_command('uname', {'-m':('armv7l', '0')}, Pihole)
+    # mock ldd to respond with aarch64 shared library
+    mock_command('ldd', {'/bin/ls':('/lib/ld-linux-armhf.so.3', '0')}, Pihole)
+    detectPlatform = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLdetect
+    ''')
+    expected_stdout = 'Detected ARM-hf architecture (armv7+)'
+    assert expected_stdout in detectPlatform.stdout
+
+def test_FTL_detect_x86_64_no_errors(Pihole):
+    ''' confirms only x86_64 package is downloaded for FTL engine '''
+    detectPlatform = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLdetect
+    ''')
+    expected_stdout = 'Detected x86_64 architecture'
+    assert expected_stdout in detectPlatform.stdout
+
+def test_FTL_detect_unknown_no_errors(Pihole):
+    ''' confirms only generic package is downloaded for FTL engine '''
+    # mock uname to return generic platform
+    mock_command('uname', {'-m':('mips', '0')}, Pihole)
+    detectPlatform = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLdetect
+    ''')
+    expected_stdout = 'Not able to detect architecture (unknown: mips)'
+    assert expected_stdout in detectPlatform.stdout
+
+def test_FTL_download_aarch64_no_errors(Pihole):
+    ''' confirms only aarch64 package is downloaded for FTL engine '''
+    # mock uname to return generic platform
+    download_binary = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLinstall pihole-FTL-aarch64-linux-gnu
+    ''')
+    expected_stdout = 'done'
+    assert expected_stdout in download_binary.stdout
+    assert 'failed' not in download_binary.stdout
+
+def test_FTL_download_unknown_fails_no_errors(Pihole):
+    ''' confirms unknown binary is not downloaded for FTL engine '''
+    # mock uname to return generic platform
+    download_binary = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLinstall pihole-FTL-mips
+    ''')
+    expected_stdout = 'failed'
+    assert expected_stdout in download_binary.stdout
+    assert 'done' not in download_binary.stdout
+
+def test_FTL_binary_installed_and_responsive_no_errors(Pihole):
+    ''' confirms FTL binary is copied and functional in installed location '''
+    installed_binary = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLdetect
+    pihole-FTL version
+    ''')
+    expected_stdout = 'v'
+    assert expected_stdout in installed_binary.stdout
+
+# def test_FTL_support_files_installed(Pihole):
+#     ''' confirms FTL support files are installed '''
+#     support_files = Pihole.run('''
+#     source /opt/pihole/basic-install.sh
+#     FTLdetect
+#     stat -c '%a %n' /var/log/pihole-FTL.log
+#     stat -c '%a %n' /run/pihole-FTL.port
+#     stat -c '%a %n' /run/pihole-FTL.pid
+#     ls -lac /run
+#     ''')
+#     assert '644 /run/pihole-FTL.port' in support_files.stdout
+#     assert '644 /run/pihole-FTL.pid' in support_files.stdout
+#     assert '644 /var/log/pihole-FTL.log' in support_files.stdout
+
 # Helper functions
 def mock_command(script, args, container):
     ''' Allows for setup of commands we don't really want to have to run for real in unit tests '''