From b2c73f8325ae059a4dec86fca42c7dead6839b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 29 May 2023 22:02:38 +0200 Subject: [PATCH 01/85] Do not try to remove stale lables on PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fe28112c..af3177d4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -29,8 +29,11 @@ jobs: operations-per-run: 300 close-issue-reason: 'not_planned' - remove_stale: # trigger "stale" removal immediately when stale issues are commented on - if: github.event_name == 'issue_comment' + remove_stale: + # trigger "stale" removal immediately when stale issues are commented on + # we need to explicitly check that the trigger does not run on comment on a PR as + # 'issue_comment' triggers on issues AND PR comments + if: github.event_name == 'issue_comment' && ${{ !github.event.issue.pull_request }} permissions: contents: read # for actions/checkout issues: write # to edit issues label From 364537b32459d3c5871f8dde0d9cad5b7e12b008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 31 May 2023 22:14:05 +0200 Subject: [PATCH 02/85] Use env variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index af3177d4..0d06ee22 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -23,7 +23,7 @@ jobs: days-before-stale: 30 days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' - stale-issue-label: $stale_label + stale-issue-label: '${{ env.stale_label }}' exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed, Documentation Needed' exempt-all-issue-assignees: true operations-per-run: 300 @@ -42,7 +42,6 @@ jobs: - name: Checkout uses: actions/checkout@v3.5.2 - name: Remove 'stale' label - run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label - env: + run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 80091232a767efd28277e90e008c643266e06577 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 12 Jul 2023 21:31:19 +0100 Subject: [PATCH 03/85] Add missing `env:`, tweak conditional Signed-off-by: Adam Warner --- .github/workflows/stale.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0d06ee22..458a7fb6 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -32,8 +32,8 @@ jobs: remove_stale: # trigger "stale" removal immediately when stale issues are commented on # we need to explicitly check that the trigger does not run on comment on a PR as - # 'issue_comment' triggers on issues AND PR comments - if: github.event_name == 'issue_comment' && ${{ !github.event.issue.pull_request }} + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issue_comment-on-issues-only-or-pull-requests-only + if: ${{ !github.event.issue.pull_request && github.event_name != 'schedule' }} permissions: contents: read # for actions/checkout issues: write # to edit issues label @@ -43,5 +43,6 @@ jobs: uses: actions/checkout@v3.5.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 7e240251d55ea5830260fc9d3bc1106d1e33bdb2 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 18 Jul 2023 08:03:05 +0100 Subject: [PATCH 04/85] Drop docker-compose from requirements.txt, it's not used. This was the only thing preventing us from using pyyaml 6.0.1 which fixes issues we were seeing with tests not running (https://github.com/yaml/pyyaml/pull/702) Signed-off-by: Adam Warner --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a4f627fa..5a551fa7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -docker-compose == 1.29.2 +pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 From 7747cd22324a28f00a4807787f2d5b6b801653b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:52:02 +0000 Subject: [PATCH 05/85] Bump tox from 4.6.4 to 4.7.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.4 to 4.7.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.4...4.7.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5a551fa7..c5d002e5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.4 +tox == 4.7.0 From 272ca8c55d0aa2f245ac1a466fd2beabca43f026 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:48:18 +0000 Subject: [PATCH 06/85] Bump tox from 4.7.0 to 4.9.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.7.0 to 4.9.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.7.0...4.9.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c5d002e5..0210aa4e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.7.0 +tox == 4.9.0 From 1c6919cf07a36c280f02dddd840e036a209747c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 10:08:08 +0000 Subject: [PATCH 07/85] Bump pytest-testinfra from 8.1.0 to 9.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 8.1.0 to 9.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/8.1.0...9.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0210aa4e..b1b36de1 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 -pytest-testinfra == 8.1.0 +pytest-testinfra == 9.0.0 tox == 4.9.0 From 924f8b88448996c8b2f11be7c11aefbb6d928e79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 10:20:12 +0000 Subject: [PATCH 08/85] Bump actions/checkout from 3.5.3 to 3.6.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.3 to 3.6.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.3...v3.6.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7fd0e4e4..1194ba0f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 43d5ca96..13b05df2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 5435b7db..60f38cf6 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8546e02b..01792fa4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From 9b68fa0b27c5b44a4f7baf211c051a6164c2c6df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:28:54 +0000 Subject: [PATCH 09/85] Bump tox from 4.9.0 to 4.10.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.9.0 to 4.10.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.9.0...4.10.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b1b36de1..90255f7f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.9.0 +tox == 4.10.0 From 9084b170cb162fdd9f615fe6b66d8e1dfa95a5b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Sep 2023 10:35:16 +0000 Subject: [PATCH 10/85] Bump tox from 4.10.0 to 4.11.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.10.0 to 4.11.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.10.0...4.11.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90255f7f..7f4d7e09 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.10.0 +tox == 4.11.1 From cdc1c1ace10e4425dde0870ffb57a3c945910057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 21 Aug 2023 21:28:17 +0200 Subject: [PATCH 11/85] Remove user agent when downloading adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/gravity.sh b/gravity.sh index 2bd21d01..db199090 100755 --- a/gravity.sh +++ b/gravity.sh @@ -401,7 +401,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain str target compression echo "" # Prepare new gravity database @@ -457,9 +457,6 @@ gravity_DownloadBlocklists() { saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" - # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) - agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs @@ -472,7 +469,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" fi echo "" done @@ -504,7 +501,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -564,7 +561,7 @@ gravity_DownloadBlocklistFromUrl() { fi # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) case $url in # Did we "download" a local file? From 843f57399c289982cbcd1629030e685b13f01f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 7 Sep 2023 22:27:49 +0200 Subject: [PATCH 12/85] Ignore ABP style entries in debug log dig test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 6f747855..ecb4fc19 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -861,11 +861,15 @@ dig_at() { local record_type="A" fi - # Find a random blocked url that has not been whitelisted. + # Find a random blocked url that has not been whitelisted and is not ABP style. # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + # Falback if no non-ABP style domains were found + if [ -z "${random_url}" ]; then + random_url="flurry.com" + fi # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is From 1f241a3d451ea6bf36eb5feb07255b8bdf91d82f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:08:40 +0000 Subject: [PATCH 13/85] Bump actions/checkout from 3.6.0 to 4.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.6.0 to 4.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.6.0...v4.0.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1194ba0f..570fabdd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 13b05df2..8d09e5e0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 60f38cf6..184319e0 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01792fa4..745a250c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From 047eac6d9c400556c107bd6fce0d28ff3f92ac4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:58:28 +0000 Subject: [PATCH 14/85] Bump tox from 4.11.1 to 4.11.3 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.1 to 4.11.3. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.1...4.11.3) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7f4d7e09..139be0fc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.11.1 +tox == 4.11.3 From 55f72ac92584342cf98870c9735554b0aa8df819 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:58:32 +0000 Subject: [PATCH 15/85] Bump pytest from 7.4.0 to 7.4.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.0 to 7.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.0...7.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7f4d7e09..9279f97d 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.0 +pytest == 7.4.2 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.1 From c3d1f366ec2b302b58488b45803a7707347e0c21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:39:24 +0000 Subject: [PATCH 16/85] Bump actions/checkout from 4.0.0 to 4.1.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 570fabdd..4685aa2c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b783ff25..14c55d10 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 184319e0..a1025629 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 745a250c..ae03db07 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From edf0060acf97e7072575599bcdbeab45541b568e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 6 Oct 2023 22:26:08 +0200 Subject: [PATCH 17/85] Fix spellcheck errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .codespellignore | 1 + advanced/Scripts/piholeDebug.sh | 4 ++-- automated install/basic-install.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.codespellignore b/.codespellignore index 501a3d67..6d05d295 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,3 +1,4 @@ doubleclick wan nwe +re-use diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index ecb4fc19..47592686 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -866,7 +866,7 @@ dig_at() { # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") - # Falback if no non-ABP style domains were found + # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" fi @@ -1503,7 +1503,7 @@ upload_to_tricorder() { # If no token was generated else # Show an error and some help instructions - # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload) + # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload) if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then : else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1f3002e7..e87edd71 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2685,7 +2685,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the dnsmasq or ftl services + # but before starting or restarting the dnsmasq or ftl services disable_resolved_stublistener # If the Web server was installed, From afa688e3a0e565e797804051bd74888712484ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 7 Oct 2023 00:21:06 +0200 Subject: [PATCH 18/85] Yu vs uh (https://www.theguardian.com/guardian-observer-style-guide-r#:~:text=re/re%2D,re%2Dsign/resign) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .codespellignore | 1 - advanced/Scripts/piholeDebug.sh | 2 +- test/test_any_automated_install.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.codespellignore b/.codespellignore index 6d05d295..501a3d67 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,4 +1,3 @@ doubleclick wan nwe -re-use diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 47592686..1ca52f45 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -449,7 +449,7 @@ os_check() { } diagnose_operating_system() { - # error message in a variable so we can easily modify it later (or re-use it) + # error message in a variable so we can easily modify it later (or reuse it) local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues." # Display the current test that is running echo_current_diagnostic "Operating system" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 36a4f0d2..840d1df7 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -70,7 +70,7 @@ def test_setupVars_are_sourced_to_global_scope(host): def test_setupVars_saved_to_file(host): """ - confirm saved settings are written to a file for future updates to re-use + confirm saved settings are written to a file for future updates to reuse """ # dedent works better with this and padding matching script below set_setup_vars = "\n" From f52b2b98630dc0f606fed94d74abee6dc2b22582 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:23:00 +0000 Subject: [PATCH 19/85] Bump actions/setup-python from 4.7.0 to 4.7.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.0 to 4.7.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.0...v4.7.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae03db07..623590da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.0 + uses: actions/setup-python@v4.7.1 with: python-version: "3.10" From 71357ecae7900b880ad336a6583491595491a350 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 14:12:02 +0100 Subject: [PATCH 20/85] adminlte->web Signed-off-by: Adam Warner --- README.md | 2 +- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/update.sh | 2 +- advanced/Scripts/updatecheck.sh | 4 ++-- advanced/Scripts/version.sh | 18 +++++++++--------- advanced/Scripts/webpage.sh | 2 +- automated install/basic-install.sh | 6 +++--- manpages/pihole.8 | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index adfd3450..eb50030b 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can read our [Core Feature Breakdown](https://docs.pi-hole.net/core/pihole-c ### The Web Interface Dashboard -This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! +This [optional dashboard](https://github.com/pi-hole/web) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! Some notable features include: diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index d69a56d3..bb94a857 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -233,7 +233,7 @@ get_sys_stats() { if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # AdminLTE not installed + # web not installed ph_lte_ver="(not installed)" ph_ftl_ver="${ph_ver_raw[1]}" else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index c41c9232..1b9997a6 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -11,7 +11,7 @@ # Please see LICENSE file for your rights under this license. # Variables -readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/AdminLTE.git" +readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git" readonly ADMIN_INTERFACE_DIR="/var/www/html/admin" readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git" readonly PI_HOLE_FILES_DIR="/etc/.pihole" diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 0cc65218..8ce69a75 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -97,10 +97,10 @@ if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" - GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" + GITHUB_WEB_VERSION="$(get_remote_version web)" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" - GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" + GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" fi diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 946c69fe..83fd0f39 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -28,7 +28,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -36,7 +36,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -44,7 +44,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -52,7 +52,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -60,13 +60,13 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } versionOutput() { - if [ "$1" = "AdminLTE" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then + if [ "$1" = "web" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then echo " WebAdmin not installed" return 1 fi @@ -117,7 +117,7 @@ defaultOutput() { versionOutput "Pi-hole" "$@" if [ "${INSTALL_WEB_INTERFACE}" = true ]; then - versionOutput "AdminLTE" "$@" + versionOutput "web" "$@" fi versionOutput "FTL" "$@" @@ -130,7 +130,7 @@ Show Pi-hole, Admin Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding AdminLTE repository + -a, --admin Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -143,7 +143,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "AdminLTE" "$@";; + "-a" | "--admin" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index c92d0458..e05a6aff 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -348,7 +348,7 @@ SetDNSServers() { IFS=',' read -r -a array <<< "${args[2]}" for index in "${!array[@]}" do - # Replace possible "\#" by "#". This fixes AdminLTE#1427 + # Replace possible "\#" by "#". This fixes web#1427 local ip ip="${array[index]//\\#/#}" diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e87edd71..4c69788f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -68,11 +68,11 @@ webroot="/var/www/html" # We clone (or update) two git repositories during the install. This helps to make sure that we always have the latest versions of the relevant files. -# AdminLTE is used to set up the Web admin interface. +# web is used to set up the Web admin interface. # Pi-hole contains various setup scripts and files which are critical to the installation. # Search for "PI_HOLE_LOCAL_REPO" in this file to see all such scripts. # Two notable scripts are gravity.sh (used to generate the HOSTS file) and advanced/Scripts/webpage.sh (used to install the Web admin interface) -webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git" +webInterfaceGitUrl="https://github.com/pi-hole/web.git" webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" @@ -334,7 +334,7 @@ package_manager_detect() { # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") - # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php + # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole web: https://www.php.net/manual/json.installation.php if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then PIHOLE_WEB_DEPS+=("${phpVer}-json") fi diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 1cf8ab35..fec1fa5e 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -212,7 +212,7 @@ Available commands and options: .br -p, --pihole Only retrieve info regarding Pi-hole repository .br - -a, --admin Only retrieve info regarding AdminLTE + -a, --admin Only retrieve info regarding web repository .br -f, --ftl Only retrieve info regarding FTL repository @@ -339,7 +339,7 @@ Displaying version information \fBpihole -v -a -c\fR .br - Display the current version of AdminLTE + Display the current version of web .br Temporarily disabling Pi-hole From 2a72012ca1448b8e2f7f0de3e79c6b9d7d72b7cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:55:06 +0000 Subject: [PATCH 21/85] Bump actions/checkout from 4.1.0 to 4.1.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.0...v4.1.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4685aa2c..6544db61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 14c55d10..0e149c79 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index a1025629..e52d4ae9 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 623590da..b070c982 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.1 From ec86124997d1b8969be2f528fc63b2626486eb73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Oct 2023 10:32:55 +0000 Subject: [PATCH 22/85] Bump pytest from 7.4.2 to 7.4.3 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.2 to 7.4.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.2...7.4.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 27417754..45c7c7c0 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.2 +pytest == 7.4.3 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From de6e61705e7d93d9c0f07ba74f343f43220770de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Nov 2023 21:40:32 +0100 Subject: [PATCH 23/85] Remove local.list and openVPN traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 1 - gravity.sh | 25 +------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 8c2388d8..55659690 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -75,7 +75,6 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" -PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" diff --git a/gravity.sh b/gravity.sh index edfe89a9..5c944454 100755 --- a/gravity.sh +++ b/gravity.sh @@ -36,9 +36,6 @@ blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" adListFile="${piholeDir}/adlists.list" -localList="${piholeDir}/local.list" -VPNList="/etc/openvpn/ipp.txt" - piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" @@ -299,12 +296,7 @@ migrate_to_database() { # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { - local lookupDomain="pi.hole" - - # Determine if $localList does not exist, and ensure it is not empty - if [[ ! -e "${localList}" ]] || [[ -s "${localList}" ]]; then - lookupDomain="raw.githubusercontent.com" - fi + local lookupDomain="raw.githubusercontent.com" # Determine if $lookupDomain is resolvable if timeout 4 getent hosts "${lookupDomain}" &> /dev/null; then @@ -686,18 +678,6 @@ gravity_ShowCount() { gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" } -# Create "localhost" entries into hosts format -gravity_generateLocalList() { - # Empty $localList if it already exists, otherwise, create it - echo "### Do not modify this file, it will be overwritten by pihole -g" > "${localList}" - chmod 644 "${localList}" - - # Add additional LAN hosts provided by OpenVPN (if available) - if [[ -f "${VPNList}" ]]; then - awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}" - fi -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -884,9 +864,6 @@ if ! gravity_DownloadBlocklists; then exit 1 fi -# Create local.list -gravity_generateLocalList - # Update gravity timestamp update_gravity_timestamp From 766cb26af5884281f343b2cba87cf7cafc207447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 22:24:34 +0100 Subject: [PATCH 24/85] Drop Fedora 36 and add Fedora 39 to the test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 2 +- test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} | 2 +- test/{tox.fedora_36.ini => tox.fedora_39.ini} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} (97%) rename test/{tox.fedora_36.ini => tox.fedora_39.ini} (80%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..9d90b2e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,9 +64,9 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_36, fedora_37, fedora_38, + fedora_39, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_36.Dockerfile b/test/_fedora_39.Dockerfile similarity index 97% rename from test/_fedora_36.Dockerfile rename to test/_fedora_39.Dockerfile index 847767e7..1727a3aa 100644 --- a/test/_fedora_36.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:39 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_39.ini similarity index 80% rename from test/tox.fedora_36.ini rename to test/tox.fedora_39.ini index 1896a45f..5c8557c9 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_39.ini @@ -1,8 +1,8 @@ [tox] envlist = py3 -[testenv:py3] +[testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 3c4f217876f9ad7f8889b94d0499a480518f8a22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:27:10 +0000 Subject: [PATCH 25/85] Bump pytest-xdist from 3.3.1 to 3.4.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.3.1 to 3.4.0. - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.3.1...v3.4.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 45c7c7c0..56423192 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.3.1 +pytest-xdist == 3.4.0 pytest-testinfra == 9.0.0 tox == 4.11.3 From 2e73eb36efbd129a105f5e18bfa0a6dd1b240032 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:54:16 +0000 Subject: [PATCH 26/85] Bump pytest-testinfra from 9.0.0 to 10.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 9.0.0 to 10.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/9.0.0...10.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 56423192..74c67fd9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.4.0 -pytest-testinfra == 9.0.0 +pytest-testinfra == 10.0.0 tox == 4.11.3 From a6565bf9a17ad4998b00ca239be9044be7f51674 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 22:07:31 +0100 Subject: [PATCH 27/85] Support special webserver.port ports ending in "s" (secure) and "r" (redirect) Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 43 ++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..000c0717 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,14 +21,31 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - PORT="$(pihole-FTL --config webserver.port)" - PORT="${PORT%%,*}" + local ports port availabilityResonse + ports="$(pihole-FTL --config webserver.port)" + port="${ports%%,*}" - availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # if the port ends in "r", it is a redirect + API_PROT="http" + # remove the "r" from the port + API_PORT="${port%?}" + else + API_PROT="http" + API_PORT="${port}" + fi - # test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then - echo "API not available at: http://localhost:${PORT}/api" + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi @@ -54,15 +71,15 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" exit 1 fi - # obtain validity and session ID from session response - validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) - SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) } DeleteSession() { @@ -70,7 +87,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -84,14 +101,14 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ]; then + if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then From 1276242a4ea3164cf2c24c9727fa7396b1495936 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 10:16:35 +0000 Subject: [PATCH 28/85] Bump pytest-xdist from 3.4.0 to 3.5.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 74c67fd9..799e3fad 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.4.0 +pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 9d57f6493751d319440727f927f099887decb886 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 10:58:21 +0000 Subject: [PATCH 29/85] Bump pytest-xdist from 3.4.0 to 3.5.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 74c67fd9..799e3fad 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.4.0 +pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 974fea592df99d97332763b6f9cf9812a6c907a4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 00:05:55 +0100 Subject: [PATCH 30/85] Iterate over ports, skip redirected ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 64 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 000c0717..b50e416a 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,26 +25,48 @@ TestAPIAvailability() { ports="$(pihole-FTL --config webserver.port)" port="${ports%%,*}" - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # if the port ends in "r", it is a redirect - API_PROT="http" - # remove the "r" from the port - API_PORT="${port%?}" - else - API_PROT="http" - API_PORT="${port}" - fi + # Iterate over comma separated list of ports + while [ "${port}" != "${ports}" ]; do + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # Ignore this port + API_PORT="0" + else + API_PROT="http" + API_PORT="${port}" + fi - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + if [ ! "${API_PORT}" = "0" ]; then + # If the port is of form "ip:port", we need to remove everything before + # the last ":" in the string, e.g., "[::]:80" -> "80" + if [ "${API_PORT#*:}" != "${API_PORT}" ]; then + API_PORT="${API_PORT##*:}" + fi - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_PORT="0" + else + # API is available at this port/protocol combination + break + fi + fi + + # remove the first port from the list + ports="${ports#*,}" + # get the next port + port="${ports%%,*}" + done + + # if API_PORT is 0, no working API port was found + if [ "${API_PORT}" = "0" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 @@ -71,7 +93,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -87,7 +109,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -101,7 +123,7 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") From 6016131280ce5f24cc53fa7989026f593f4d2d4f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 22:59:49 +0100 Subject: [PATCH 31/85] Ensure we also check the last port Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index b50e416a..2952fb43 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -26,16 +26,18 @@ TestAPIAvailability() { port="${ports%%,*}" # Iterate over comma separated list of ports - while [ "${port}" != "${ports}" ]; do + while [ -n "${ports}" ]; do # if the port ends with an "s", it is a secure connection if [ "${port#"${port%?}"}" = "s" ]; then # remove the "s" from the port API_PROT="https" API_PORT="${port%?}" elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port + # Ignore this port, the client may not be able to follow the + # redirected target when FTL is not used as local resolver API_PORT="0" else + # otherwise it is an insecure (plain HTTP) connection API_PROT="http" API_PORT="${port}" fi @@ -50,8 +52,9 @@ TestAPIAvailability() { API_URL="${API_PROT}://localhost:${API_PORT}/api" availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination API_PORT="0" else # API is available at this port/protocol combination @@ -59,9 +62,9 @@ TestAPIAvailability() { fi fi - # remove the first port from the list + # If the loop has not been broken, remove the first port from the list + # and get the next port ports="${ports#*,}" - # get the next port port="${ports%%,*}" done @@ -192,3 +195,6 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } + + +TestAPIAvailability From 96bf07863f6bdd4d9cad91d5444fef5f85d47b61 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 09:10:06 +0100 Subject: [PATCH 32/85] Use CHAOS TXT local.api.txt instead of trying to parse pihole-FTL --config webserver.ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 90 +++++++++++++++++++-------------------- advanced/Scripts/query.sh | 6 +-- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 2952fb43..46da37cd 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,62 +21,60 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - local ports port availabilityResonse - ports="$(pihole-FTL --config webserver.port)" - port="${ports%%,*}" + local chaos_api_list availabilityResonse - # Iterate over comma separated list of ports - while [ -n "${ports}" ]; do - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port, the client may not be able to follow the - # redirected target when FTL is not used as local resolver - API_PORT="0" + # Query the API URLs from FTL using CHAOS TXT local.api.ftl + # The result is a space-separated enumeration of full URLs + # e.g., "http://localhost:80/api" "https://localhost:443/api" + chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" + + # If the query was not successful, the variable is empty + if [ -z "${chaos_api_list}" ]; then + echo "API not available. Please check connectivity" + exit 1 + fi + + # Iterate over space-separated list of URLs + while [ -n "${chaos_api_list}" ]; do + # Get the first URL + API_URL="${chaos_api_list%% *}" + # Strip leading and trailing quotes + API_URL="${API_URL%\"}" + API_URL="${API_URL#\"}" + + # Test if the API is available at this URL + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") + + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination + API_PORT="" else - # otherwise it is an insecure (plain HTTP) connection - API_PROT="http" - API_PORT="${port}" + # API is available at this URL combination + break fi - if [ ! "${API_PORT}" = "0" ]; then - # If the port is of form "ip:port", we need to remove everything before - # the last ":" in the string, e.g., "[::]:80" -> "80" - if [ "${API_PORT#*:}" != "${API_PORT}" ]; then - API_PORT="${API_PORT##*:}" - fi + # Remove the first URL from the list + local last_api_list + last_api_list="${chaos_api_list}" + chaos_api_list="${chaos_api_list#* }" - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then - # API is not available at this port/protocol combination - API_PORT="0" - else - # API is available at this port/protocol combination - break - fi + # If the list did not change, we are at the last element + if [ "${last_api_list}" = "${chaos_api_list}" ]; then + # Remove the last element + chaos_api_list="" fi - - # If the loop has not been broken, remove the first port from the list - # and get the next port - ports="${ports#*,}" - port="${ports%%,*}" done - # if API_PORT is 0, no working API port was found - if [ "${API_PORT}" = "0" ]; then + # if API_PORT is empty, no working API port was found + if [ -n "${API_PORT}" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi } -Authenthication() { +Authentication() { # Try to authenticate LoginAPI @@ -96,7 +94,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -108,11 +106,11 @@ LoginAPI() { } DeleteSession() { - # if a valid Session exists (no password required or successful authenthication) and - # SID is not null (successful authenthication only), delete the session + # if a valid Session exists (no password required or successful Authentication) and + # SID is not null (successful Authentication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..62d29d5b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -121,14 +121,14 @@ Main(){ # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. # Therefore, we try to query directly without authentication but do authenticat if 401 is returned - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") if [ "${data}" = 401 ]; then # Unauthenticated, so authenticate with the FTL server required - Authenthication + Authentication # send query again - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") fi GenerateOutput "${data}" From 29d010dc2ccc04cd6563d8154415e2070b5bd9a7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 10:21:02 +0100 Subject: [PATCH 33/85] Use files.gravity_tmp as temporary directory for the intermediate lists Signed-off-by: DL6ER --- gravity.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..8dc8edd8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -41,6 +41,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) +GRAVITY_TMPDIR=$(getFTLConfigValue files.gravity_tmp) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" @@ -48,9 +49,7 @@ domainsExtension="domains" curl_connect_timeout=10 -# Set up tmp dir variable in case it's not configured -: "${GRAVITY_TMPDIR:=/tmp}" - +# Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" GRAVITY_TMPDIR="/tmp" From bfc824f2ff05f56d24697230463a3ec0bf2a25cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:04:50 +0000 Subject: [PATCH 34/85] Bump tox from 4.11.3 to 4.11.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.3 to 4.11.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.3...4.11.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 799e3fad..bfc6d027 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.3 +tox == 4.11.4 From f5fe550a2e27e5b81ffbe55d3f209c890dda3b94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:35:21 +0000 Subject: [PATCH 35/85] Bump tox from 4.11.3 to 4.11.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.3 to 4.11.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.3...4.11.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 799e3fad..bfc6d027 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.3 +tox == 4.11.4 From 32a741b5c74bab31ed5c02f5ba33f65f631fea7f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 2 Dec 2023 22:42:36 +0100 Subject: [PATCH 36/85] We do not follow 308 but FTL also doesn't suggest it Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 46da37cd..b7bc2a86 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,7 +25,7 @@ TestAPIAvailability() { # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs - # e.g., "http://localhost:80/api" "https://localhost:443/api" + # e.g., "http://localhost:80/api/" "https://localhost:443/api/" chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" # If the query was not successful, the variable is empty @@ -45,8 +45,8 @@ TestAPIAvailability() { # Test if the API is available at this URL availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # Test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then # API is not available at this port/protocol combination API_PORT="" else @@ -131,7 +131,7 @@ GetFTLData() { # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then + if [ "${status}" = 200 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then @@ -193,6 +193,3 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } - - -TestAPIAvailability From 2681835f94ec072e183d6cd6de267f25b93fb0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 00:21:04 +0100 Subject: [PATCH 37/85] Treat FTL return data as strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 2 +- advanced/Scripts/query.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..bf6e3654 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -93,7 +93,7 @@ GetFTLData() { if [ "${status}" = 200 ]; then # response OK - echo "${data}" + printf %s "${data}" elif [ "${status}" = 000 ]; then # connection lost echo "000" diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..789efe75 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -46,14 +46,14 @@ GenerateOutput(){ data="${1}" # construct a new json for the list results where each object contains the domain and the related type - lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') + lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json - num_gravity=$(echo "${gravity_data}" | jq length ) - num_lists=$(echo "${lists_data}" | jq length ) + num_gravity=$(printf %s "${gravity_data}" | jq length ) + num_lists=$(printf %s "${lists_data}" | jq length ) if [ "${partial}" = true ]; then search_type_str="partially" @@ -66,7 +66,7 @@ GenerateOutput(){ if [ "${num_lists}" -gt 0 ]; then # Convert the data to a csv, each line is a "domain,type" string # not using jq's @csv here as it quotes each value individually - lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) + lists_data_csv=$(printf %s "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) # Generate output for each csv line, separating line in a domain and type substring at the ',' echo "${lists_data_csv}" | while read -r line; do @@ -79,7 +79,7 @@ GenerateOutput(){ if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do From 159817b7e286c4015030bfe61c6100b7d5b2c028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 20:35:26 +0100 Subject: [PATCH 38/85] Use development-v6 as ftl branch for binary test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 7d9d15f6..62588c0a 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -465,7 +465,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From 16180e4b234510d3c52545f817164fd9873448a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:36:44 +0000 Subject: [PATCH 39/85] Bump actions/setup-python from 4.7.1 to 5.0.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b070c982..35ed0c30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: "3.10" From c34464d1e8f6e0ba5731171791d9a31d801dfa5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:36:49 +0000 Subject: [PATCH 40/85] Bump actions/stale from 8.0.0 to 9.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8.0.0...v9.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0e149c79..095d7358 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 2db2a25d..96650818 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From 8a71e4253e0733fec4a4cb805ea219ff8ec8dab2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:18 +0000 Subject: [PATCH 41/85] Bump actions/stale from 8.0.0 to 9.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8.0.0...v9.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d9de09d2..74ee9bef 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 2db2a25d..96650818 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From e49d7fa5f1dd19a80c798318a270e80f0a6d9415 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:42 +0000 Subject: [PATCH 42/85] Bump actions/setup-python from 4.7.1 to 5.0.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..ec4da6bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: "3.10" From a87d1bbc4fcf107753324f551ac54bfe7bc45a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:22:43 +0100 Subject: [PATCH 43/85] Remove pihole.sudo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole.sudo | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 advanced/Templates/pihole.sudo diff --git a/advanced/Templates/pihole.sudo b/advanced/Templates/pihole.sudo deleted file mode 100644 index 708309be..00000000 --- a/advanced/Templates/pihole.sudo +++ /dev/null @@ -1,9 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Allows the WebUI to use Pi-hole commands -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -# From 54186a63eee39a2e420dda8d170807e3492bbc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:33:13 +0100 Subject: [PATCH 44/85] Remove test for sudo file as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 62588c0a..c53070dc 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -174,10 +174,6 @@ def test_installPihole_fresh_install_readableFiles(host): ) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - # check not readable sudoers file - check_sudo = test_cmd.format("r", "/etc/sudoers.d/pihole", piholeuser) - actual_rc = host.run(check_sudo).rc - assert exit_status_success != actual_rc # check not readable cron file check_sudo = test_cmd.format("x", "/etc/cron.d/", piholeuser) actual_rc = host.run(check_sudo).rc From 274d4c263c2ec4c8ef3d3b4a55bae45aa3fb694b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:23:39 +0100 Subject: [PATCH 45/85] Add bash-completion to PIHOLE_DEPS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 72f00992..f5bf15e6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw bash-completion) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -349,7 +349,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw bash-completion) # If neither apt-get or yum/dnf package managers were found else From fe7299323c389ca86fe941ecadaee6ce082695a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:40:03 +0100 Subject: [PATCH 46/85] Update bash_completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 305a3f5b..b2740724 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -7,17 +7,13 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; - "admin") - opts_admin="celsius fahrenheit interface kelvin password privacylevel" - COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) ) - ;; "checkout") opts_checkout="core ftl web master dev" COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) @@ -31,7 +27,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) ;; "query") - opts_query="-adlist -all -exact" + opts_query="--partial --all" COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) ;; "updatePihole"|"-up") @@ -41,23 +37,7 @@ _pihole() { "version") opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; - "interface") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)" - COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) ) - else - return 1 - fi - ;; - "privacylevel") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_privacy="0 1 2 3" - COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) ) - else - return 1 - fi - ;; + ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From 6d02d4056f6abd63ff3d9479979502b6523f715e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:01:30 +0100 Subject: [PATCH 47/85] Rename option --admin to --web in version function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- advanced/bash-completion/pihole | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index e3b4a6ae..2983c04e 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -24,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; + "web" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -32,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; + "web" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -40,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -48,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -56,7 +56,7 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; + "web" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } @@ -114,11 +114,11 @@ defaultOutput() { helpFunc() { echo "Usage: pihole -v [repo | option] [option] Example: 'pihole -v -p -l' -Show Pi-hole, Admin Console & FTL versions +Show Pi-hole, Web Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding web repository + -w, --web Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -131,7 +131,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "web" "$@";; + "-w" | "--web" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index b2740724..4fe8f83a 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -35,7 +35,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; "version") - opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" + opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) ;; "core"|"admin"|"ftl") From c90a27c5096d53f4be21044fdd71cb380806bc3f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2023 23:06:50 +0100 Subject: [PATCH 48/85] Add "-ni" to all sqlite3 invocations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 34 ++++++++-------- advanced/Scripts/list.sh | 20 +++++----- advanced/Scripts/piholeARPTable.sh | 4 +- advanced/Scripts/piholeDebug.sh | 12 +++--- advanced/Scripts/piholeLogFlush.sh | 2 +- gravity.sh | 39 +++++++++---------- 6 files changed, 55 insertions(+), 56 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index e36d9b1e..7c4deaa7 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -19,13 +19,13 @@ upgrade_gravityDB(){ auditFile="${piholeDir}/auditlog.list" # Get database version - version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -40,28 +40,28 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script unifies the formally separated domain # lists into a single table with a UNIQUE domain constraint echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" version=4 fi if [[ "$version" == "4" ]]; then # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" version=5 fi if [[ "$version" == "5" ]]; then # This migration script upgrades the adlist view # to return an ID used in gravity.sh echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" version=6 fi if [[ "$version" == "6" ]]; then @@ -69,7 +69,7 @@ upgrade_gravityDB(){ # which is automatically associated to all clients not # having their own group assignments echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" version=7 fi if [[ "$version" == "7" ]]; then @@ -77,21 +77,21 @@ upgrade_gravityDB(){ # to ensure uniqueness on the group name # We also add date_added and date_modified columns echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" version=8 fi if [[ "$version" == "8" ]]; then # This migration fixes some issues that were introduced # in the previous migration script. echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" version=9 fi if [[ "$version" == "9" ]]; then # This migration drops unused tables and creates triggers to remove # obsolete groups assignments when the linked items are deleted echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" version=10 fi if [[ "$version" == "10" ]]; then @@ -101,44 +101,44 @@ upgrade_gravityDB(){ # to keep the copying process generic (needs the same columns in both the # source and the destination databases). echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" version=11 fi if [[ "$version" == "11" ]]; then # Rename group 0 from "Unassociated" to "Default" echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi if [[ "$version" == "12" ]]; then # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi if [[ "$version" == "13" ]]; then # Add columns number and status to adlist table echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi if [[ "$version" == "14" ]]; then # Changes the vw_adlist created in 5_to_6 echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi if [[ "$version" == "15" ]]; then # Add column abp_entries to adlist table echo -e " ${INFO} Upgrading gravity database from version 15 to 16" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi if [[ "$version" == "16" ]]; then # Add antigravity table # Add column type to adlist table (to support adlist types) echo -e " ${INFO} Upgrading gravity database from version 16 to 17" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b76a7ef7..76558e58 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -150,18 +150,18 @@ AddDomain() { domain="$1" # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" if [[ "${existingTypeId}" == "${typeId}" ]]; then if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" fi else existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" fi @@ -177,10 +177,10 @@ AddDomain() { # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" else # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" fi } @@ -189,7 +189,7 @@ RemoveDomain() { domain="$1" # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" requestedListname="$(GetListnameFromTypeId "${typeId}")" @@ -206,14 +206,14 @@ RemoveDomain() { fi reload=true # Remove it from the current list - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" } Displaylist() { local count num_pipes domain enabled status nicedate requestedListname requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" + data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty list" @@ -251,10 +251,10 @@ Displaylist() { } NukeList() { - count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") + count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 5daa025d..b92dd124 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -39,7 +39,7 @@ flushARP(){ # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" echo " Database location: ${DBFILE}" echo " Output: ${output}" @@ -47,7 +47,7 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 931e95a4..aa0e61c4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -718,7 +718,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" @@ -1064,7 +1064,7 @@ show_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1089,7 +1089,7 @@ show_FTL_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1155,7 +1155,7 @@ analyze_gravity_list() { fi show_db_entries "Info table" "SELECT property,value FROM info" "20 40" - gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" + gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" gravity_updated="$(date -d @"${gravity_updated_raw}")" log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}" log_write "" @@ -1163,7 +1163,7 @@ analyze_gravity_list() { OLD_IFS="$IFS" IFS=$'\r\n' local gravity_sample=() - mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}" for line in "${gravity_sample[@]}"; do @@ -1195,7 +1195,7 @@ database_integrity_check(){ log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)" unset result - result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" + result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" if [[ -z ${result} ]]; then log_write "${TICK} No foreign key errors in ${database}" else diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 14542e4b..4d97fec5 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -63,7 +63,7 @@ else fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) - deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") + deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") # Restart pihole-FTL to force reloading history sudo pihole restartdns diff --git a/gravity.sh b/gravity.sh index 8dc8edd8..f51103ff 100755 --- a/gravity.sh +++ b/gravity.sh @@ -48,7 +48,6 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 - # Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" @@ -64,7 +63,7 @@ gravityOLDfile="${gravityDIR}/gravity_old.db" # Generate new SQLite3 file from schema template generate_gravity_database() { - if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi @@ -79,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -118,7 +117,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -163,7 +162,7 @@ database_table_from_file() { # Get MAX(id) from domainlist when INSERTing into this table if [[ "${table}" == "domainlist" ]]; then - rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" if [[ -z "$rowid" ]]; then rowid=0 fi @@ -193,7 +192,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -213,7 +212,7 @@ database_table_from_file() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -228,7 +227,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -244,7 +243,7 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -362,9 +361,9 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" + mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -393,7 +392,7 @@ gravity_DownloadBlocklists() { str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." rm "${gravityTEMPfile}" > /dev/null 2>&1 - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -413,7 +412,7 @@ gravity_DownloadBlocklists() { copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" fi - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -663,12 +662,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -749,7 +748,7 @@ database_recovery() { local str="Checking integrity of existing gravity database (this can take a while)" local option="${1}" echo -ne " ${INFO} ${str}..." - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" if [[ ${result} = "ok" ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -757,7 +756,7 @@ database_recovery() { str="Checking foreign keys of existing gravity database (this can take a while)" echo -ne " ${INFO} ${str}..." unset result - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" if [[ -z ${result} ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" if [[ "${option}" != "force" ]]; then @@ -776,7 +775,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." # We have to remove any possibly existing recovery database or this will fail rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" From 3f7413d538dae3524b0a5d627668aecd60bcd815 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2023 23:07:35 +0100 Subject: [PATCH 49/85] Add "-ni" to all sqlite3 invocations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 30 ++++++++-------- advanced/Scripts/list.sh | 20 +++++------ advanced/Scripts/piholeARPTable.sh | 4 +-- advanced/Scripts/piholeDebug.sh | 12 +++---- advanced/Scripts/piholeLogFlush.sh | 2 +- advanced/Scripts/query.sh | 6 ++-- advanced/Scripts/webpage.sh | 12 +++---- gravity.sh | 36 +++++++++---------- 8 files changed, 61 insertions(+), 61 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index a7ba60a9..1459ecd9 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -19,13 +19,13 @@ upgrade_gravityDB(){ auditFile="${piholeDir}/auditlog.list" # Get database version - version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -40,28 +40,28 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script unifies the formally separated domain # lists into a single table with a UNIQUE domain constraint echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" version=4 fi if [[ "$version" == "4" ]]; then # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" version=5 fi if [[ "$version" == "5" ]]; then # This migration script upgrades the adlist view # to return an ID used in gravity.sh echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" version=6 fi if [[ "$version" == "6" ]]; then @@ -69,7 +69,7 @@ upgrade_gravityDB(){ # which is automatically associated to all clients not # having their own group assignments echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" version=7 fi if [[ "$version" == "7" ]]; then @@ -77,21 +77,21 @@ upgrade_gravityDB(){ # to ensure uniqueness on the group name # We also add date_added and date_modified columns echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" version=8 fi if [[ "$version" == "8" ]]; then # This migration fixes some issues that were introduced # in the previous migration script. echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" version=9 fi if [[ "$version" == "9" ]]; then # This migration drops unused tables and creates triggers to remove # obsolete groups assignments when the linked items are deleted echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" version=10 fi if [[ "$version" == "10" ]]; then @@ -101,31 +101,31 @@ upgrade_gravityDB(){ # to keep the copying process generic (needs the same columns in both the # source and the destination databases). echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" version=11 fi if [[ "$version" == "11" ]]; then # Rename group 0 from "Unassociated" to "Default" echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi if [[ "$version" == "12" ]]; then # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi if [[ "$version" == "13" ]]; then # Add columns number and status to adlist table echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi if [[ "$version" == "14" ]]; then # Changes the vw_adlist created in 5_to_6 echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b76a7ef7..76558e58 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -150,18 +150,18 @@ AddDomain() { domain="$1" # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" if [[ "${existingTypeId}" == "${typeId}" ]]; then if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" fi else existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" fi @@ -177,10 +177,10 @@ AddDomain() { # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" else # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" fi } @@ -189,7 +189,7 @@ RemoveDomain() { domain="$1" # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" requestedListname="$(GetListnameFromTypeId "${typeId}")" @@ -206,14 +206,14 @@ RemoveDomain() { fi reload=true # Remove it from the current list - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" } Displaylist() { local count num_pipes domain enabled status nicedate requestedListname requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" + data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty list" @@ -251,10 +251,10 @@ Displaylist() { } NukeList() { - count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") + count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 5daa025d..b92dd124 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -39,7 +39,7 @@ flushARP(){ # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" echo " Database location: ${DBFILE}" echo " Output: ${output}" @@ -47,7 +47,7 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1ca52f45..fb1efbeb 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -865,7 +865,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" @@ -1226,7 +1226,7 @@ show_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1251,7 +1251,7 @@ show_FTL_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1317,7 +1317,7 @@ analyze_gravity_list() { fi show_db_entries "Info table" "SELECT property,value FROM info" "20 40" - gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" + gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" gravity_updated="$(date -d @"${gravity_updated_raw}")" log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}" log_write "" @@ -1325,7 +1325,7 @@ analyze_gravity_list() { OLD_IFS="$IFS" IFS=$'\r\n' local gravity_sample=() - mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}" for line in "${gravity_sample[@]}"; do @@ -1357,7 +1357,7 @@ database_integrity_check(){ log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)" unset result - result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" + result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" if [[ -z ${result} ]]; then log_write "${TICK} No foreign key errors in ${database}" else diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 3473fad5..b06aac8b 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -63,7 +63,7 @@ else fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) - deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") + deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") # Restart pihole-FTL to force reloading history sudo pihole restartdns diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 1d3b0a29..ebcc6f79 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -96,7 +96,7 @@ scanDatabaseTable() { # Are there ABP entries on gravity? # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')" - abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null + abpfound="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null # Create search string for ABP entries only if needed if [ "${abpfound}" -eq 1 ]; then @@ -129,7 +129,7 @@ scanDatabaseTable() { fi # Send prepared query to gravity database - result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null + result="$(pihole-FTL sqlite3 -ni -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null if [[ -z "${result}" ]]; then # Return early when there are no matches in this table return @@ -166,7 +166,7 @@ scanRegexDatabaseTable() { list_type="${3:-}" # Query all regex from the corresponding database tables - mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null) + mapfile -t regexList < <(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index e05a6aff..62ab4ea9 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -560,13 +560,13 @@ CustomizeAdLists() { if CheckUrl "${address}"; then if [[ "${args[2]}" == "enable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" elif [[ "${args[2]}" == "del" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" else echo "Not permitted" return 1 @@ -700,12 +700,12 @@ addAudit() done # Insert only the domain here. The date_added field will be # filled with its default value (date_added = current timestamp) - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" } clearAudit() { - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domain_audit;" } SetPrivacyLevel() { diff --git a/gravity.sh b/gravity.sh index db199090..62631e3c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -84,7 +84,7 @@ fi # Generate new SQLite3 file from schema template generate_gravity_database() { - if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi @@ -99,7 +99,7 @@ gravity_swap_databases() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -135,7 +135,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -179,7 +179,7 @@ database_table_from_file() { # Get MAX(id) from domainlist when INSERTing into this table if [[ "${table}" == "domainlist" ]]; then - rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" if [[ -z "$rowid" ]]; then rowid=0 fi @@ -209,7 +209,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -229,7 +229,7 @@ database_table_from_file() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -244,7 +244,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -260,7 +260,7 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -378,8 +378,8 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -408,7 +408,7 @@ gravity_DownloadBlocklists() { str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." rm "${gravityTEMPfile}" > /dev/null 2>&1 - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -428,7 +428,7 @@ gravity_DownloadBlocklists() { copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" fi - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -668,12 +668,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -754,7 +754,7 @@ database_recovery() { local str="Checking integrity of existing gravity database (this can take a while)" local option="${1}" echo -ne " ${INFO} ${str}..." - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" if [[ ${result} = "ok" ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -762,7 +762,7 @@ database_recovery() { str="Checking foreign keys of existing gravity database (this can take a while)" echo -ne " ${INFO} ${str}..." unset result - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" if [[ -z ${result} ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" if [[ "${option}" != "force" ]]; then @@ -781,7 +781,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." # We have to remove any possibly existing recovery database or this will fail rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" From fe4d934a406cf4fa45a72668170868bd57448537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:31:36 +0100 Subject: [PATCH 50/85] Simplify pihole -v MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 138 ++++++++---------------------------- pihole | 5 +- 2 files changed, 31 insertions(+), 112 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 2983c04e..ff7f0815 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,6 +8,10 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# Ignore warning about `local` being undefinded in POSIX +# shellcheck disable=SC3043 +# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + # Source the versions file poupulated by updatechecker.sh cachedVersions="/etc/pihole/versions" @@ -21,118 +25,34 @@ else . "$cachedVersions" fi -getLocalVersion() { - case ${1} in - "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; - "FTL" ) echo "${FTL_VERSION:=N/A}";; - esac -} +main() { + local details + details=$1 -getLocalHash() { - case ${1} in - "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; - "FTL" ) echo "${FTL_HASH:=N/A}";; - esac -} - -getRemoteHash(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; - esac -} - -getRemoteVersion(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; - esac -} - -getLocalBranch(){ - case ${1} in - "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; - "FTL" ) echo "${FTL_BRANCH:=N/A}";; - esac -} - -versionOutput() { - - [ "$2" = "-c" ] || [ "$2" = "--current" ] || [ -z "$2" ] && current=$(getLocalVersion "${1}") && branch=$(getLocalBranch "${1}") - [ "$2" = "-l" ] || [ "$2" = "--latest" ] || [ -z "$2" ] && latest=$(getRemoteVersion "${1}") - if [ "$2" = "--hash" ]; then - [ "$3" = "-c" ] || [ "$3" = "--current" ] || [ -z "$3" ] && curHash=$(getLocalHash "${1}") && branch=$(getLocalBranch "${1}") - [ "$3" = "-l" ] || [ "$3" = "--latest" ] || [ -z "$3" ] && latHash=$(getRemoteHash "${1}") && branch=$(getLocalBranch "${1}") - fi - - # We do not want to show the branch name when we are on master, - # blank out the variable in this case - if [ "$branch" = "master" ]; then - branch="" + if [ "${details}" = true ]; then + echo "Core" + echo " Version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo " Branch is ${CORE_BRANCH:=N/A}" + echo " Hash is ${CORE_HASH:=N/A} (Latest: ${GITHUB_CORE_HASH:=N/A})" + echo "Web" + echo " Version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo " Branch is ${WEB_BRANCH:=N/A}" + echo " Hash is ${WEB_HASH:=N/A} (Latest: ${GITHUB_WEB_HASH:=N/A})" + echo "FTL" + echo " Version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" + echo " Branch is ${FTL_BRANCH:=N/A}" + echo " Hash is ${FTL_HASH:=N/A} (Latest: ${GITHUB_FTL_HASH:=N/A})" else - branch="$branch " + echo "Core version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo "Web version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo "FTL version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" fi - - if [ -n "$current" ] && [ -n "$latest" ]; then - output="${1} version is $branch$current (Latest: $latest)" - elif [ -n "$current" ] && [ -z "$latest" ]; then - output="Current ${1} version is $branch$current" - elif [ -z "$current" ] && [ -n "$latest" ]; then - output="Latest ${1} version is $latest" - elif [ -n "$curHash" ] && [ -n "$latHash" ]; then - output="Local ${1} hash is $curHash (Remote: $latHash)" - elif [ -n "$curHash" ] && [ -z "$latHash" ]; then - output="Current local ${1} hash is $curHash" - elif [ -z "$curHash" ] && [ -n "$latHash" ]; then - output="Latest remote ${1} hash is $latHash" - elif [ -z "$curHash" ] && [ -z "$latHash" ]; then - output="Hashes for ${1} not available" - else - errorOutput - return 1 - fi - - [ -n "$output" ] && echo " $output" } -errorOutput() { - echo " Invalid Option! Try 'pihole -v --help' for more information." - exit 1 -} - -defaultOutput() { - versionOutput "Pi-hole" "$@" - versionOutput "web" "$@" - versionOutput "FTL" "$@" -} - -helpFunc() { - echo "Usage: pihole -v [repo | option] [option] -Example: 'pihole -v -p -l' -Show Pi-hole, Web Console & FTL versions - -Repositories: - -p, --pihole Only retrieve info regarding Pi-hole repository - -w, --web Only retrieve info regarding web repository - -f, --ftl Only retrieve info regarding FTL repository - -Options: - -c, --current Return the current version - -l, --latest Return the latest version - --hash Return the GitHub hash from your local repositories - -h, --help Show this help dialog" - exit 0 -} - -case "${1}" in - "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-w" | "--web" ) shift; versionOutput "web" "$@";; - "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; - "-h" | "--help" ) helpFunc;; - * ) defaultOutput "$@";; +# Process options (if present) +case "$1" in + "-vv" ) details=true;; + * ) details=false;; esac + +main "${details}" diff --git a/pihole b/pihole index 47da4ddd..ef93146c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,6 @@ uninstallFunc() { } versionFunc() { - shift exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" } @@ -508,7 +507,7 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Add '-h' for more info on version usage + Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -531,7 +530,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "version" ) versionFunc "$@";; + "-v" | "-vv" | "version" ) versionFunc "$@";; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From b333e30162a07287bcf313ec9da01e1644c72d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:55:01 +0100 Subject: [PATCH 51/85] Remove version options from bash completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 4 ---- 1 file changed, 4 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 4fe8f83a..89e02d2f 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -34,10 +34,6 @@ _pihole() { opts_update="--check-only" COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; - "version") - opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" - COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From c3c31a1a60dcf68ae3b08a2c474485fdbd3c9cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 23:09:58 +0100 Subject: [PATCH 52/85] Print version details automatically if not on master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- pihole | 5 ++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index ff7f0815..2422ea6d 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -27,7 +27,13 @@ fi main() { local details - details=$1 + details=false + + # Automatically show detailed information if + # at least one of the components is not on master branch + if [ ! "${CORE_BRANCH}" = "master" ] || [ ! "${WEB_BRANCH}" = "master" ] || [ ! "${FTL_BRANCH}" = "master" ] ; then + details=true + fi if [ "${details}" = true ]; then echo "Core" @@ -49,10 +55,4 @@ main() { fi } -# Process options (if present) -case "$1" in - "-vv" ) details=true;; - * ) details=false;; -esac - -main "${details}" +main diff --git a/pihole b/pihole index ef93146c..7c84771c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,7 @@ uninstallFunc() { } versionFunc() { - exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" + exec "${PI_HOLE_SCRIPT_DIR}"/version.sh } restartDNS() { @@ -507,7 +507,6 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -530,7 +529,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "-vv" | "version" ) versionFunc "$@";; + "-v" | "version" ) versionFunc;; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From e8338d059009f009f8b93461e4fee51792acb600 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Dec 2023 10:41:00 +0100 Subject: [PATCH 53/85] Install FTL's development branch to get the latest features during tests Signed-off-by: DL6ER --- test/test_any_automated_install.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 840d1df7..c1b91664 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -176,6 +176,12 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) + # Install FTL's development branch to get the latest features + host.run( + """ + echo "development" > /etc/pihole/ftlbranch + """ + ) install = host.run( """ export TERM=xterm @@ -431,6 +437,12 @@ def test_installPihole_fresh_install_readableBlockpage(host, test_webpage): setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) + # Install FTL's development branch to get the latest features + host.run( + """ + echo "development" > /etc/pihole/ftlbranch + """ + ) installWeb = host.run( """ export TERM=xterm From 0cfcdc4b50e16612bf7572f290c1960c0f39e873 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:08:32 +0000 Subject: [PATCH 54/85] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6544db61..fc821194 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,12 +29,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: 'python' - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From c6049d400214d45f6bd9229e6fd3291a3d53f3dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:30:20 +0000 Subject: [PATCH 55/85] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6544db61..fc821194 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,12 +29,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: 'python' - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 0066c6fbffa889525751478468b1cfba77272e20 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:29:11 +0100 Subject: [PATCH 56/85] Make IDs of anti-/gravity lists available in vw_(anti)gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 6 +++++ .../database_migration/gravity/17_to_18.sql | 25 +++++++++++++++++++ advanced/Templates/gravity.db.sql | 6 ++--- gravity.sh | 2 +- 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/17_to_18.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7c4deaa7..b263b40d 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -141,4 +141,10 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi + if [[ "$version" == "17" ]]; then + # Add adlist.id to vw_gravity and vw_antigravity + echo -e " ${INFO} Upgrading gravity database from version 17 to 18" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" + version=18 + fi } diff --git a/advanced/Scripts/database_migration/gravity/17_to_18.sql b/advanced/Scripts/database_migration/gravity/17_to_18.sql new file mode 100644 index 00000000..00171a9a --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/17_to_18.sql @@ -0,0 +1,25 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP VIEW vw_gravity; +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM gravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id + LEFT JOIN adlist ON adlist.id = gravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); + +DROP VIEW vw_antigravity; +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +UPDATE info SET value = 18 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 46f26ba7..097b0a78 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','17'); +INSERT INTO "info" VALUES('version','18'); CREATE TABLE domain_audit ( @@ -144,14 +144,14 @@ CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist AND domainlist.type = 3 ORDER BY domainlist.id; -CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM gravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id LEFT JOIN adlist ON adlist.id = gravity.adlist_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM antigravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id LEFT JOIN adlist ON adlist.id = antigravity.adlist_id diff --git a/gravity.sh b/gravity.sh index f51103ff..ebe7e740 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) status="$?" if [[ "${status}" -ne 0 ]]; then From 8e8c7ecad212d45548dc89f408d2545d332a37a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:41:43 +0100 Subject: [PATCH 57/85] Run gravity upgrade on checkout/update Signed-off-by: DL6ER --- automated install/basic-install.sh | 5 +++++ gravity.sh | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..5863e818 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2225,6 +2225,11 @@ main() { # but before starting or restarting the ftl service disable_resolved_stublistener + # Check if gravity database needs to be upgraded. If so, do it without rebuilding + # gravity altogether. This may be a very long running task needlessly blocking + # the update process. + /opt/pihole/gravity.sh --upgrade + printf " %b Restarting services...\\n" "${INFO}" # Start services diff --git a/gravity.sh b/gravity.sh index ebe7e740..216a67a5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -831,6 +831,7 @@ for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; "-r" | "--repair" ) repairSelector "$3";; + "-u" | "--upgrade" ) upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"; exit 0;; "-h" | "--help" ) helpFunc;; esac done From 1b87ec067ef8d19df0cc6af9f4aa30ba69696d3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 06:12:18 +0100 Subject: [PATCH 58/85] Exit early if the database does not exist (e.g. in some CI tests) Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index b263b40d..7b356b3a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -18,6 +18,11 @@ upgrade_gravityDB(){ piholeDir="${2}" auditFile="${piholeDir}/auditlog.list" + # Exit early if the database does not exist (e.g. in CI tests) + if [[ ! -f "${database}" ]]; then + return + fi + # Get database version version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" From 0c6363572b2ddde8c44f3bd5d06dfff92f929380 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 22:00:49 +0100 Subject: [PATCH 59/85] Append the interface to the gateway address if it is a link-local address Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index aa0e61c4..a78f94ee 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -547,17 +547,24 @@ ping_gateway() { ping_ipv4_or_ipv6 "${protocol}" # Check if we are using IPv4 or IPv6 # Find the default gateways using IPv4 or IPv6 - local gateway + local gateway gateway_addr gateway_iface log_write "${INFO} Default IPv${protocol} gateway(s):" while IFS= read -r gateway; do - log_write " ${gateway}" - done < <(ip -"${protocol}" route | grep default | cut -d ' ' -f 3) + log_write " $(cut -d ' ' -f 3 <<< "${gateway}")%$(cut -d ' ' -f 5 <<< "${gateway}")" + done < <(ip -"${protocol}" route | grep default) - gateway=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_addr=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_iface=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 5 | head -n 1) # If there was at least one gateway - if [ -n "${gateway}" ]; then + if [ -n "${gateway_addr}" ]; then + # Append the interface to the gateway address if it is a link-local address + if [[ "${gateway_addr}" =~ ^fe80 ]]; then + gateway="${gateway_addr}%${gateway_iface}" + else + gateway="${gateway_addr}" + fi # Let the user know we will ping the gateway for a response log_write " * Pinging first gateway ${gateway}..." # Try to quietly ping the gateway 3 times, with a timeout of 3 seconds, using numeric output only, From 00340136bdc517cc71d4458fb5ccd8d784724767 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Dec 2023 21:35:24 +0100 Subject: [PATCH 60/85] Extend %iface logic to the dig test. Also fix intentation in this function Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 39 +++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a78f94ee..2c3ebb14 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -764,24 +764,29 @@ dig_at() { # Removes CIDR and everything thereafter (e.g., scope properties) addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")" if [ -n "${addresses}" ]; then - while IFS= read -r local_address ; do + while IFS= read -r local_address ; do + # If ${local_address} is an IPv6 link-local address, append the interface name to it + if [[ "${local_address}" =~ ^fe80 ]]; then + local_address="${local_address}%${iface}" + fi + # Check if Pi-hole can use itself to block a domain - if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then - # If it can, show success - if [[ "${local_dig}" == *"status: NOERROR"* ]]; then - local_dig="NOERROR" - elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then - local_dig="NXDOMAIN" - else - # Extract the first entry in the answer section from dig's output, - # replacing any multiple spaces and tabs with a single space - local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" - fi - log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" - else - # Otherwise, show a failure - log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" - fi + if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then + # If it can, show success + if [[ "${local_dig}" == *"status: NOERROR"* ]]; then + local_dig="NOERROR" + elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then + local_dig="NXDOMAIN" + else + # Extract the first entry in the answer section from dig's output, + # replacing any multiple spaces and tabs with a single space + local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" + fi + log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" + else + # Otherwise, show a failure + log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" + fi done <<< "${addresses}" else log_write "${TICK} No IPv${protocol} address available on ${COL_CYAN}${iface}${COL_NC}" From d2828310f240aed098e5b3d787f5d63f47575a7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Dec 2023 11:12:20 +0100 Subject: [PATCH 61/85] Modify DELETE triggers to delete BEFORE instead of AFTER to prevent possible foreign key constraint violations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/18_to_19.sql | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 advanced/Scripts/database_migration/gravity/18_to_19.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7b356b3a..e99f1df2 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -152,4 +152,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" version=18 fi + if [[ "$version" == "18" ]]; then + # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent + # foreign key constraint violations + echo -e " ${INFO} Upgrading gravity database from version 18 to 19" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" + version=19 + fi } diff --git a/advanced/Scripts/database_migration/gravity/18_to_19.sql b/advanced/Scripts/database_migration/gravity/18_to_19.sql new file mode 100644 index 00000000..c85a4d57 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/18_to_19.sql @@ -0,0 +1,27 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP TRIGGER tr_domainlist_delete; +CREATE TRIGGER tr_domainlist_delete BEFORE DELETE ON domainlist + BEGIN + DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id; + END; + +DROP TRIGGER tr_adlist_delete; +CREATE TRIGGER tr_adlist_delete BEFORE DELETE ON adlist + BEGIN + DELETE FROM adlist_by_group WHERE adlist_id = OLD.id; + END; + +DROP TRIGGER tr_client_delete; +CREATE TRIGGER tr_client_delete BEFORE DELETE ON client + BEGIN + DELETE FROM client_by_group WHERE client_id = OLD.id; + END; + +UPDATE info SET value = 19 WHERE property = 'version'; + +COMMIT; From 9c3578856f1c47887d7af5f1bc972ce44f5316ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 10:02:39 +0000 Subject: [PATCH 62/85] Bump pytest from 7.4.3 to 7.4.4 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.3 to 7.4.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.3...7.4.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index bfc6d027..0873d097 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.3 +pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.4 From 05e7d0ee9263e17e0bc6706b3822310d7ed084a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 10:10:22 +0000 Subject: [PATCH 63/85] Bump pytest from 7.4.3 to 7.4.4 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.3 to 7.4.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.3...7.4.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index bfc6d027..0873d097 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.3 +pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.4 From 220c0675ef67adea60161fdd38f409736bcaaac0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 7 Jan 2024 21:03:48 +0000 Subject: [PATCH 64/85] As we do in other tests, specify `development-v6` branch of FTL is downloaded for the tests. We need to change this to `develoment` once v6 is released Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c53070dc..ab301a6e 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -239,6 +239,7 @@ def test_FTL_detect_aarch64_no_errors(host): mock_command("uname", {"-m": ("aarch64", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -272,6 +273,7 @@ def test_FTL_detect_armv6_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -305,6 +307,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -338,6 +341,7 @@ def test_FTL_detect_armv7_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -371,6 +375,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -393,6 +398,7 @@ def test_FTL_detect_x86_64_no_errors(host): """ detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -415,6 +421,7 @@ def test_FTL_detect_unknown_no_errors(host): mock_command("uname", {"-m": ("mips", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -442,6 +449,7 @@ def test_FTL_download_aarch64_no_errors(host): ) download_binary = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user FTLinstall "pihole-FTL-aarch64-linux-gnu" @@ -458,6 +466,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) From df9c60e3511f7871c1392f19bebbef1112be64ac Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 8 Jan 2024 18:26:59 +0000 Subject: [PATCH 65/85] Reduce code duplication in FTL arch detect tests. Use parametrize to run the same test with different parameters for each arch we need to test. Also include detection of unusupported in this test. Create FTL_BRANCH constant to be used in functions (less places to remember to change it) Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 253 +++++------------------------ 1 file changed, 37 insertions(+), 216 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index ab301a6e..0930f0af 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -12,6 +12,8 @@ from .conftest import ( run_script, ) +FTL_BRANCH = "development-v6" + def test_supported_package_manager(host): """ @@ -80,11 +82,7 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") # Workaround to get FTLv6 installed until it reaches master branch - host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - """ - ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') install = host.run( """ export TERM=xterm @@ -231,49 +229,37 @@ def test_update_package_cache_failure_no_errors(host): assert "Error: Unable to update package cache." in updateCache.stdout -def test_FTL_detect_aarch64_no_errors(host): +@pytest.mark.parametrize( + "arch,detected_string,supported", + [ + ("aarch64", "AArch64 (64 Bit ARM)", True), + ("armv6", "ARMv6", True), + ("armv7l", "ARMv7 (or newer)", True), + ("armv7", "ARMv7 (or newer)", True), + ("armv8a", "ARMv7 (or newer)", True), + ("x86_64", "x86_64", True), + ("riscv64", "riscv64", True), + ("mips", "mips", False), + ], +) +def test_FTL_detect_no_errors(host, arch, detected_string, supported): """ - confirms only aarch64 package is downloaded for FTL engine + confirms only correct package is downloaded for FTL engine """ - # mock uname to return aarch64 platform - mock_command("uname", {"-m": ("aarch64", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_no_errors(host): - """ - confirms only armv6 package is downloaded for FTL engine - """ - # mock uname to return armv6 platform - mock_command("uname", {"-m": ("armv6", "0")}, host) - # mock readelf to respond with armv6l CPU architecture + # mock uname to return passed platform + mock_command("uname", {"-m": (arch, "0")}, host) + # mock readelf to respond with passed CPU architecture mock_command_2( "readelf", { - "-A /bin/sh": ("Tag_CPU_arch: armv6", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6", "0"), + "-A /bin/sh": ("Tag_CPU_arch: " + arch, "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: " + arch, "0"), }, host, ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') detectPlatform = host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -282,195 +268,30 @@ def test_FTL_detect_armv6_no_errors(host): FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv6 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7l_no_errors(host): - """ - confirms only armv7l package is downloaded for FTL engine - """ - # mock uname to return armv7l platform - mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock readelf to respond with armv7l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7l", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7_no_errors(host): - """ - confirms only armv7 package is downloaded for FTL engine - """ - # mock uname to return armv7 platform - mock_command("uname", {"-m": ("armv7", "0")}, host) - # mock readelf to respond with armv7 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv8a_no_errors(host): - """ - confirms only armv8a package is downloaded for FTL engine - """ - # mock uname to return armv8a platform - mock_command("uname", {"-m": ("armv8a", "0")}, host) - # mock readelf to respond with armv8a CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv8a", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv8a", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv7 (or newer) architecture (armv8a)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_x86_64_no_errors(host): - """ - confirms only x86_64 package is downloaded for FTL engine - """ - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected x86_64 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_unknown_no_errors(host): - """confirms only generic package is downloaded for FTL engine""" - # mock uname to return generic platform - mock_command("uname", {"-m": ("mips", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = "Not able to detect architecture (unknown: mips)" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_download_aarch64_no_errors(host): - """ - confirms only aarch64 package is downloaded for FTL engine - """ - # mock dialog answers and ensure installer dependencies - mock_command("dialog", {"*": ("", "0")}, host) - host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - install_dependent_packages ${INSTALLER_DEPS[@]} - """ - ) - download_binary = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - FTLinstall "pihole-FTL-aarch64-linux-gnu" - """ - ) - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in download_binary.stdout - assert "error" not in download_binary.stdout.lower() + if supported: + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected " + detected_string + " architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + else: + expected_stdout = ( + "Not able to detect architecture (unknown: " + detected_string + ")" + ) + assert expected_stdout in detectPlatform.stdout def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ confirms FTL development binary is copied and functional in installed location """ + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From debab10792c7bee8cd6ba18b6c3798c4f03c8a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 7 Jan 2024 21:37:58 +0100 Subject: [PATCH 66/85] Use 204 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index fe73a608..18a48ce7 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -113,9 +113,8 @@ DeleteSession() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; + "204") printf "%b" "Session successfully deleted.\n";; "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; - "410") printf "%b" "Session successfully deleted.\n";; esac; fi From 9eb47319f572674c91ce0a49b5240ddb66aee027 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jan 2024 10:32:36 +0000 Subject: [PATCH 67/85] Bump tox from 4.11.4 to 4.12.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.4 to 4.12.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.4...4.12.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0873d097..59fd13cc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.4 +tox == 4.12.0 From b662fd6f0101193a37b79377b077ffa37c33dfc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jan 2024 10:43:13 +0000 Subject: [PATCH 68/85] Bump tox from 4.11.4 to 4.12.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.4 to 4.12.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.4...4.12.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0873d097..59fd13cc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.4 +tox == 4.12.0 From 47f06dfd71824f199af2a9b757f6ac06f9724ef4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 Jan 2024 20:53:34 +0100 Subject: [PATCH 69/85] Do not store remote version in versions file if on custom branch. It's always wrong here Signed-off-by: DL6ER --- advanced/Scripts/updatecheck.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 66f1a7ab..d2a338a7 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -26,7 +26,12 @@ function get_local_hash() { } function get_remote_version() { - curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + # if ${2} is = "master" we need to use the "latest" endpoint, otherwise, we simply return null + if [[ "${2}" == "master" ]]; then + curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + else + echo "null" + fi } @@ -73,7 +78,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "CORE_BRANCH" "${CORE_BRANCH}" CORE_HASH="$(get_local_hash /etc/.pihole)" addOrEditKeyValPair "${VERSION_FILE}" "CORE_HASH" "${CORE_HASH}" -GITHUB_CORE_VERSION="$(get_remote_version pi-hole)" +GITHUB_CORE_VERSION="$(get_remote_version pi-hole "${CORE_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_VERSION" "${GITHUB_CORE_VERSION}" GITHUB_CORE_HASH="$(get_remote_hash pi-hole "${CORE_BRANCH}")" @@ -91,7 +96,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" -GITHUB_WEB_VERSION="$(get_remote_version web)" +GITHUB_WEB_VERSION="$(get_remote_version web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" @@ -108,7 +113,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "FTL_BRANCH" "${FTL_BRANCH}" FTL_HASH="$(pihole-FTL --hash)" addOrEditKeyValPair "${VERSION_FILE}" "FTL_HASH" "${FTL_HASH}" -GITHUB_FTL_VERSION="$(get_remote_version FTL)" +GITHUB_FTL_VERSION="$(get_remote_version FTL "${FTL_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_FTL_VERSION" "${GITHUB_FTL_VERSION}" GITHUB_FTL_HASH="$(get_remote_hash FTL "${FTL_BRANCH}")" From ba2682c907e446138f142c64b2bef4a617e79692 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 10:38:21 +0000 Subject: [PATCH 70/85] Bump tox from 4.12.0 to 4.12.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.0 to 4.12.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.0...4.12.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 59fd13cc..dea4ace9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.0 +tox == 4.12.1 From 60de50bb73f3df5f78c68726c3cd31ba1d5193ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 10:49:27 +0000 Subject: [PATCH 71/85] Bump tox from 4.12.0 to 4.12.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.0 to 4.12.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.0...4.12.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 59fd13cc..dea4ace9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.0 +tox == 4.12.1 From 935a4ce0b3262fa3b3d468f018771a5001ffe145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jan 2024 16:56:57 +0100 Subject: [PATCH 72/85] Also remove Fedora 37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 +-- test/_fedora_37.Dockerfile | 18 ------------------ test/tox.fedora_37.ini | 8 -------- 3 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 test/_fedora_37.Dockerfile delete mode 100644 test/tox.fedora_37.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d90b2e7..eadfc930 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,8 +64,7 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_37, - fedora_38, + fedora_38, fedora_39, ] env: diff --git a/test/_fedora_37.Dockerfile b/test/_fedora_37.Dockerfile deleted file mode 100644 index b4f939ba..00000000 --- a/test/_fedora_37.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM fedora:37 -RUN dnf install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini deleted file mode 100644 index 9c8752cc..00000000 --- a/test/tox.fedora_37.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 31a8f150b2e3a5411871f1163c7cc7a433e7ecc5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Jan 2024 19:09:14 +0100 Subject: [PATCH 73/85] Add checking for availability of ftl.pi-hole.net when using FTL from a custom branch. If the server is down (or the user is offline, cannot resolve the domain, etc.), we fail early and hard instead of possibly corrupting the installation Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 22 +++++++++++----- advanced/Scripts/update.sh | 7 ++++- automated install/basic-install.sh | 41 +++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 39d39b1c..41fd8606 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -164,7 +164,9 @@ checkout() { path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" - if check_download_exists "$path"; then + check_download_exists "$path" + local ret=$? + if [ $ret -eq 0 ]; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch chmod 644 /etc/pihole/ftlbranch @@ -175,11 +177,19 @@ checkout() { # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else - echo " ${CROSS} Requested branch \"${2}\" is not available" - ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) - echo -e " ${INFO} Available branches for FTL are:" - for e in "${ftlbranches[@]}"; do echo " - $e"; done - exit 1 + if [[ $ret -eq 1 ]]; then + echo " ${CROSS} Requested branch \"${2}\" is not available" + ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) + echo -e " ${INFO} Available branches for FTL are:" + for e in "${ftlbranches[@]}"; do echo " - $e"; done + exit 1 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + exit 1 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + exit 1 + fi fi else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 9dae66df..8a35ef2e 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -144,7 +144,7 @@ main() { local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) - if FTLcheckUpdate "${binary}" > /dev/null; then + if FTLcheckUpdate "${binary}"; then FTL_update=true echo -e " ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}" else @@ -155,8 +155,13 @@ main() { 2) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Branch is not available.${COL_NC}\\n\\t\\t\\tUse ${COL_LIGHT_GREEN}pihole checkout ftl [branchname]${COL_NC} to switch to a valid branch." ;; + 3) + echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, cannot reach download server${COL_NC}" + exit 1 + ;; *) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, contact support${COL_NC}" + exit 1 esac FTL_update=false fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..af7ca151 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1687,12 +1687,19 @@ update_dialogs() { } check_download_exists() { + # Check if the download exists and we can reach the server status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) - if grep -q "404" <<< "$status"; then - return 1 - else + + # Check the status code + if grep -q "200" <<< "$status"; then return 0 + elif grep -q "404" <<< "$status"; then + return 1 fi + + # Other error or no status code at all, e.g., no Internet, server not + # available/reachable, ... + return 2 } fully_fetch_repo() { @@ -1957,10 +1964,8 @@ get_binary_name() { } FTLcheckUpdate() { - #In the next section we check to see if FTL is already installed (in case of pihole -r). - #If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download - printf " %b Checking for existing FTL binary...\\n" "${INFO}" - + # In the next section we check to see if FTL is already installed (in case of pihole -r). + # If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download local ftlLoc ftlLoc=$(command -v pihole-FTL 2>/dev/null) @@ -1979,14 +1984,24 @@ FTLcheckUpdate() { local localSha1 if [[ ! "${ftlBranch}" == "master" ]]; then - #Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! + # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! local path path="${ftlBranch}/${binary}" # shellcheck disable=SC1090 - if ! check_download_exists "$path"; then - printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" - printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" - return 2 + check_download_exists "$path" + local ret=$? + if [ $ret -ne 0 ]; then + if [[ $ret -eq 1 ]]; then + printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" + printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" + return 2 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + return 3 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + return 4 + fi fi if [[ ${ftlLoc} ]]; then @@ -2011,12 +2026,14 @@ FTLcheckUpdate() { FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag + # Get the latest version from the GitHub API if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location: | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then # There was an issue while retrieving the latest version printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3 fi + # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From 6e946f76d6e0166d6f2d0e86ed7ab01686faf6bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Feb 2024 10:21:48 +0000 Subject: [PATCH 74/85] Bump pytest from 7.4.4 to 8.0.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.4 to 8.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.4...8.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index dea4ace9..2b8dfb1b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.4 +pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.12.1 From f8bfd59f11332b664c954ebfb2ed1586fbd3d070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 22:24:34 +0100 Subject: [PATCH 75/85] Drop Fedora 36 and add Fedora 39 to the test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 2 +- test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} | 2 +- test/{tox.fedora_36.ini => tox.fedora_39.ini} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} (97%) rename test/{tox.fedora_36.ini => tox.fedora_39.ini} (81%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35ed0c30..67377255 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,9 +57,9 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_36, fedora_37, fedora_38, + fedora_39, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_36.Dockerfile b/test/_fedora_39.Dockerfile similarity index 97% rename from test/_fedora_36.Dockerfile rename to test/_fedora_39.Dockerfile index 847767e7..1727a3aa 100644 --- a/test/_fedora_36.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:39 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_39.ini similarity index 81% rename from test/tox.fedora_36.ini rename to test/tox.fedora_39.ini index 515487ed..7a538371 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_39.ini @@ -1,8 +1,8 @@ [tox] envlist = py3 -[testenv:py3] +[testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py From 7b6f0d1596f879b65552c5deddc0dc620b290d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jan 2024 16:56:57 +0100 Subject: [PATCH 76/85] Also remove Fedora 37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 +-- test/_fedora_37.Dockerfile | 18 ------------------ test/tox.fedora_37.ini | 8 -------- 3 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 test/_fedora_37.Dockerfile delete mode 100644 test/tox.fedora_37.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67377255..fabf380e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,8 +57,7 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_37, - fedora_38, + fedora_38, fedora_39, ] env: diff --git a/test/_fedora_37.Dockerfile b/test/_fedora_37.Dockerfile deleted file mode 100644 index b4f939ba..00000000 --- a/test/_fedora_37.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM fedora:37 -RUN dnf install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini deleted file mode 100644 index 2a8ef398..00000000 --- a/test/tox.fedora_37.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py From f0878c0890926e8f7443c7349581b1b10c8d3e3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 21:39:17 +0000 Subject: [PATCH 77/85] Bump pytest from 7.4.4 to 8.0.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.4 to 8.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.4...8.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index dea4ace9..2b8dfb1b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.4 +pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.12.1 From b5ab8ac1980442f19d474f630f47da6dd851f151 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 16:54:22 +0100 Subject: [PATCH 78/85] Change UNIQUEness constraint from (address) to (address, type) in the adlist table. This will allow certain adlists to be associated to different groups. A possible scenario is an adlist meant to block a specific service (e.g. Twitter, Youtube, etc.). It can then either be used to ensure these services are really blocked on the devices of group A but will never be blocked on devices of group B. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 097b0a78..42060443 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -27,7 +27,7 @@ CREATE TABLE domainlist CREATE TABLE adlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, + address TEXT NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), @@ -37,7 +37,8 @@ CREATE TABLE adlist invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, abp_entries INTEGER NOT NULL DEFAULT 0, - type INTEGER NOT NULL DEFAULT 0 + type INTEGER NOT NULL DEFAULT 0, + UNIQUE(address, type) ); CREATE TABLE adlist_by_group From bcb712b6e3f408b0080768b9c2c77c2a4c7c6dbd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 21:07:15 +0100 Subject: [PATCH 79/85] Fix accidential double -ni from one of the last PRs Signed-off-by: DL6ER --- gravity.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 592aa801..4d785d8a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -114,7 +114,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}"; } 2>&1) + output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -352,8 +352,8 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" + mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" # Parse source domains from $sources @@ -667,12 +667,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi From 75fadb9b55fbb1454bbddd60d0ed99924200d2d4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 08:55:26 +0100 Subject: [PATCH 80/85] Adlists need to be grouped by both address and type to differentiate between anti-/gravity lists Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 581d2069..df7db893 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -48,7 +48,7 @@ GenerateOutput() { lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) From 9ff43040ec52d9bc1adefa2f4a69775835dfc67c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 09:58:23 +0100 Subject: [PATCH 81/85] Add list type in pihole -q Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index df7db893..493c75ea 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -41,14 +41,14 @@ Options: GenerateOutput() { local data gravity_data lists_data num_gravity num_lists search_type_str - local gravity_data_csv lists_data_csv line current_domain + local gravity_data_csv lists_data_csv line current_domain url type color data="${1}" # construct a new json for the list results where each object contains the domain and the related type lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), type: (.[0].type), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) @@ -78,15 +78,27 @@ GenerateOutput() { if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")') + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .type, .domains[]] | join(",")') # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do + # Get first part of the line, the URL + url=${line%%,*} + + # cut off URL, leaving "type,domain,domain,...." + line=${line#*,} + type=${line%%,*} + # type == "block" -> red, type == "allow" -> green + if [ "${type}" = "block" ]; then + color="${COL_RED}" + else + color="${COL_GREEN}" + fi # print adlist URL - printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}" + printf "%s (%s)\n\n" " - ${COL_BLUE}${url}${COL_NC}" "${color}${type}${COL_NC}" - # cut off URL, leaving "domain,domain,...." + # cut off type, leaving "domain,domain,...." line=${line#*,} # print each domain and remove it from the string until nothing is left while [ ${#line} -gt 0 ]; do From b322f1e98bf1ec64b705add4f089cda22ae8e5e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Feb 2024 10:50:41 +0000 Subject: [PATCH 82/85] Bump tox from 4.12.1 to 4.13.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.1 to 4.13.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.1...4.13.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2b8dfb1b..a6826838 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.1 +tox == 4.13.0 From 3ed29f494b93be8b03bc9739ad5f03a7cf578a25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:45:22 +0000 Subject: [PATCH 83/85] Bump pytest from 8.0.0 to 8.0.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.0 to 8.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.0...8.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a6826838..3e341719 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.0 +pytest == 8.0.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.13.0 From 07e6c0d250febddf706e5b414466b88a6c89b288 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:49:07 +0000 Subject: [PATCH 84/85] Bump pytest-testinfra from 10.0.0 to 10.1.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.0.0 to 10.1.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.0.0...10.1.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 3e341719..2a559b05 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.0.1 pytest-xdist == 3.5.0 -pytest-testinfra == 10.0.0 +pytest-testinfra == 10.1.0 tox == 4.13.0 From fd1372df3e72509e16c92bfcb9b4489d9b232fc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 10:01:31 +0000 Subject: [PATCH 85/85] Bump pytest from 8.0.1 to 8.0.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.1 to 8.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.1...8.0.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2a559b05..ffb05813 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.1 +pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.13.0