diff --git a/blockbot/composer.json b/blockbot/composer.json
index f13a2d17..40583e20 100644
--- a/blockbot/composer.json
+++ b/blockbot/composer.json
@@ -1,24 +1,24 @@
{
- "name": "friendica-addons/blockbot",
- "description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.",
- "type": "friendica-addon",
- "authors": [
- {
- "name": "Philipp Holzer",
- "email": "admin@philipp.info",
- "homepage": "https://friendica.philipp.info/profile/nupplaphil",
- "role": "Developer"
- }
- ],
- "require": {
- "php": ">=5.6.0",
- "jaybizzle/crawler-detect": "1.*"
- },
- "license": "3-clause BSD license",
- "minimum-stability": "stable",
- "config": {
- "optimize-autoloader": true,
- "autoloader-suffix": "BlockBotAddon",
- "preferred-install": "dist"
- }
+ "name": "friendica-addons/blockbot",
+ "description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.",
+ "type": "friendica-addon",
+ "authors": [
+ {
+ "name": "Philipp Holzer",
+ "email": "admin@philipp.info",
+ "homepage": "https://friendica.philipp.info/profile/nupplaphil",
+ "role": "Developer"
+ }
+ ],
+ "require": {
+ "php": ">=5.6.0",
+ "jaybizzle/crawler-detect": "1.*"
+ },
+ "license": "3-clause BSD license",
+ "minimum-stability": "stable",
+ "config": {
+ "optimize-autoloader": true,
+ "autoloader-suffix": "BlockBotAddon",
+ "preferred-install": "dist"
+ }
}
diff --git a/blockbot/composer.lock b/blockbot/composer.lock
index 26b021b1..50f71a37 100644
--- a/blockbot/composer.lock
+++ b/blockbot/composer.lock
@@ -8,24 +8,23 @@
"packages": [
{
"name": "jaybizzle/crawler-detect",
- "version": "v1.2.80",
+ "version": "v1.2.116",
"source": {
"type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git",
- "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847"
+ "reference": "97e9fe30219e60092e107651abb379a38b342921"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
- "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
+ "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
+ "reference": "97e9fe30219e60092e107651abb379a38b342921",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
- "phpunit/phpunit": "^4.8|^5.5|^6.5",
- "satooshi/php-coveralls": "1.*"
+ "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
},
"type": "library",
"autoload": {
@@ -53,7 +52,7 @@
"crawlerdetect",
"php crawler detect"
],
- "time": "2019-04-05T19:52:02+00:00"
+ "time": "2023-07-21T15:49:49+00:00"
}
],
"packages-dev": [],
@@ -65,5 +64,6 @@
"platform": {
"php": ">=5.6.0"
},
- "platform-dev": []
+ "platform-dev": [],
+ "plugin-api-version": "1.1.0"
}
diff --git a/blockbot/vendor/composer/ClassLoader.php b/blockbot/vendor/composer/ClassLoader.php
index 95f7e097..03b9bb9c 100644
--- a/blockbot/vendor/composer/ClassLoader.php
+++ b/blockbot/vendor/composer/ClassLoader.php
@@ -60,7 +60,7 @@ class ClassLoader
public function getPrefixes()
{
if (!empty($this->prefixesPsr0)) {
- return call_user_func_array('array_merge', $this->prefixesPsr0);
+ return call_user_func_array('array_merge', array_values($this->prefixesPsr0));
}
return array();
@@ -279,7 +279,7 @@ class ClassLoader
*/
public function setApcuPrefix($apcuPrefix)
{
- $this->apcuPrefix = function_exists('apcu_fetch') && ini_get('apc.enabled') ? $apcuPrefix : null;
+ $this->apcuPrefix = function_exists('apcu_fetch') && filter_var(ini_get('apc.enabled'), FILTER_VALIDATE_BOOLEAN) ? $apcuPrefix : null;
}
/**
diff --git a/blockbot/vendor/composer/autoload_real.php b/blockbot/vendor/composer/autoload_real.php
index ccb886a2..404079e0 100644
--- a/blockbot/vendor/composer/autoload_real.php
+++ b/blockbot/vendor/composer/autoload_real.php
@@ -13,6 +13,9 @@ class ComposerAutoloaderInitBlockBotAddon
}
}
+ /**
+ * @return \Composer\Autoload\ClassLoader
+ */
public static function getLoader()
{
if (null !== self::$loader) {
diff --git a/blockbot/vendor/composer/installed.json b/blockbot/vendor/composer/installed.json
index d255d573..db9064b6 100644
--- a/blockbot/vendor/composer/installed.json
+++ b/blockbot/vendor/composer/installed.json
@@ -1,27 +1,26 @@
[
{
"name": "jaybizzle/crawler-detect",
- "version": "v1.2.80",
- "version_normalized": "1.2.80.0",
+ "version": "v1.2.116",
+ "version_normalized": "1.2.116.0",
"source": {
"type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git",
- "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847"
+ "reference": "97e9fe30219e60092e107651abb379a38b342921"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
- "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
+ "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
+ "reference": "97e9fe30219e60092e107651abb379a38b342921",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
- "phpunit/phpunit": "^4.8|^5.5|^6.5",
- "satooshi/php-coveralls": "1.*"
+ "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
},
- "time": "2019-04-05T19:52:02+00:00",
+ "time": "2023-07-21T15:49:49+00:00",
"type": "library",
"installation-source": "dist",
"autoload": {
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml
new file mode 100644
index 00000000..1c083c40
--- /dev/null
+++ b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml
@@ -0,0 +1,23 @@
+name: Check & fix styling
+
+on: [ push ]
+
+jobs:
+ php-cs-fixer:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.head_ref }}
+
+ - name: Run PHP CS Fixer
+ uses: docker://oskarstark/php-cs-fixer-ga:2.18.6
+ with:
+ args: --config=.php_cs.dist --allow-risky=yes
+
+ - name: Commit changes
+ uses: stefanzweifel/git-auto-commit-action@v4
+ with:
+ commit_message: Fix styling
\ No newline at end of file
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml
new file mode 100644
index 00000000..22911114
--- /dev/null
+++ b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml
@@ -0,0 +1,56 @@
+name: Test
+
+on:
+ push:
+ branches:
+ - "master"
+ pull_request:
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: true
+ matrix:
+ php: [5.3, 5.4, 5.5, 5.6, 7.0, 7.1, 7.2, 7.3, 7.4, 8.0, 8.1, 8.2]
+
+ name: PHP:${{ matrix.php }}
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Setup PHP, with composer
+ uses: shivammathur/setup-php@v2
+ with:
+ php-version: ${{ matrix.php }}
+ tools: composer:v2
+ coverage: xdebug
+
+ - name: Get composer cache directory
+ id: composer-cache
+ run: |
+ echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+ shell: bash
+
+ - name: Cache composer dependencies
+ uses: actions/cache@v3
+ with:
+ path: ${{ steps.composer-cache.outputs.dir }}
+ key: dependencies-php-${{ matrix.php }}-composer-${{ hashFiles('composer.json') }}
+ restore-keys: dependencies-php-${{ matrix.php }}-composer-
+
+ - name: Install Composer dependencies
+ run: |
+ composer install --prefer-dist --no-interaction --no-suggest
+
+ - name: Run Unit tests
+ run: |
+ vendor/bin/phpunit --coverage-clover=tests/logs/clover.xml
+
+ - name: Upload coverage results to Coveralls
+ env:
+ COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ composer global require php-coveralls/php-coveralls "^1.0"
+ coveralls --coverage_clover=tests/logs/clover.xml -v
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist b/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist
new file mode 100644
index 00000000..91c91af9
--- /dev/null
+++ b/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist
@@ -0,0 +1,33 @@
+in([
+ __DIR__.'/src',
+ __DIR__.'/tests',
+ ])
+ ->name('*.php')
+ ->ignoreDotFiles(true)
+ ->ignoreVCS(true);
+
+return PhpCsFixer\Config::create()
+ ->setRules([
+ '@PSR2' => true,
+ 'array_syntax' => ['syntax' => 'long'],
+ 'ordered_imports' => ['sortAlgorithm' => 'alpha'],
+ 'no_unused_imports' => true,
+ 'not_operator_with_successor_space' => true,
+ 'trailing_comma_in_multiline_array' => true,
+ 'phpdoc_scalar' => true,
+ 'unary_operator_spaces' => true,
+ 'binary_operator_spaces' => true,
+ 'blank_line_before_statement' => [
+ 'statements' => ['break', 'continue', 'declare', 'return', 'throw', 'try'],
+ ],
+ 'phpdoc_single_line_var_spacing' => true,
+ 'phpdoc_var_without_name' => true,
+ 'method_argument_space' => [
+ 'on_multiline' => 'ensure_fully_multiline',
+ 'keep_multiple_spaces_after_comma' => true,
+ ],
+ ])
+ ->setFinder($finder);
\ No newline at end of file
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/LICENSE b/blockbot/vendor/jaybizzle/crawler-detect/LICENSE
index 2f4e15e2..569c7b4f 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/LICENSE
+++ b/blockbot/vendor/jaybizzle/crawler-detect/LICENSE
@@ -1,6 +1,6 @@
The MIT License (MIT)
-Copyright (c) 2015-2018 Mark Beech
+Copyright (c) 2015-2020 Mark Beech
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/README.md b/blockbot/vendor/jaybizzle/crawler-detect/README.md
index e7c25f3b..57ec8e8b 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/README.md
+++ b/blockbot/vendor/jaybizzle/crawler-detect/README.md
@@ -1,24 +1,23 @@
-
-crawlerdetect.io
+
+crawlerdetect.io
-
-
+
-
-
## About CrawlerDetect
-CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the user agent and http_from header. Currently able to detect 1,000's of bots/spiders/crawlers.
+CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the `user agent` and `http_from` header. Currently able to detect 1,000's of bots/spiders/crawlers.
### Installation
-Run `composer require jaybizzle/crawler-detect 1.*` or add `"jaybizzle/crawler-detect" :"1.*"` to your `composer.json`.
+```
+composer require jaybizzle/crawler-detect
+```
### Usage
```PHP
@@ -46,7 +45,7 @@ If you find a bot/spider/crawler user agent that CrawlerDetect fails to detect,
Failing that, just create an issue with the user agent you have found, and we'll take it from there :)
### Laravel Package
-If you would like to use this with Laravel 4/5, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect)
+If you would like to use this with Laravel, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect)
### Symfony Bundle
To use this library with Symfony 2/3/4, check out the [CrawlerDetectBundle](https://github.com/nicolasmure/CrawlerDetectBundle).
@@ -57,16 +56,21 @@ To use this library with the YII2 framework, check out [yii2-crawler-detect](htt
### ES6 Library
To use this library with NodeJS or any ES6 application based, check out [es6-crawler-detect](https://github.com/JefferyHus/es6-crawler-detect).
+### Python Library
+To use this library in a Python project, check out [crawlerdetect](https://github.com/moskrc/CrawlerDetect).
+
+### JVM Library (written in Java)
+To use this library in a JVM project (including Java, Scala, Kotlin, etc.), check out [CrawlerDetect](https://github.com/nekosoftllc/crawler-detect).
+
### .NET Library
To use this library in a .net standard (including .net core) based project, check out [NetCrawlerDetect](https://github.com/gplumb/NetCrawlerDetect).
-### Nette Extension
-To use this library with the Nette framework, checkout [NetteCrawlerDetect](https://github.com/JanGalek/Crawler-Detect).
-
### Ruby Gem
-
To use this library with Ruby on Rails or any Ruby-based application, check out [crawler_detect](https://github.com/loadkpi/crawler_detect) gem.
+### Go Module
+To use this library with Go, check out the [crawlerdetect](https://github.com/x-way/crawlerdetect) module.
+
_Parts of this class are based on the brilliant [MobileDetect](https://github.com/serbanghita/Mobile-Detect)_
[](https://github.com/JayBizzle/Crawler-Detect)
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/composer.json b/blockbot/vendor/jaybizzle/crawler-detect/composer.json
index 0c0babe6..4774117e 100755
--- a/blockbot/vendor/jaybizzle/crawler-detect/composer.json
+++ b/blockbot/vendor/jaybizzle/crawler-detect/composer.json
@@ -16,8 +16,7 @@
"php": ">=5.3.0"
},
"require-dev": {
- "phpunit/phpunit": "^4.8|^5.5|^6.5",
- "satooshi/php-coveralls": "1.*"
+ "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
},
"autoload": {
"psr-4": {
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/export.php b/blockbot/vendor/jaybizzle/crawler-detect/export.php
index 4c4b9d5d..6c7459c4 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/export.php
+++ b/blockbot/vendor/jaybizzle/crawler-detect/export.php
@@ -37,5 +37,5 @@ function outputJson($object)
function outputTxt($object)
{
$className = (new ReflectionClass($object))->getShortName();
- file_put_contents("raw/$className.txt", implode($object->getAll(), PHP_EOL));
+ file_put_contents("raw/$className.txt", implode(PHP_EOL, $object->getAll()));
}
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json
index a1e690eb..003b87c8 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json
+++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json
@@ -1 +1 @@
-[".*Java.*outbrain"," YLT","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^DangDang","^DavClnt","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^HTTPClient\\\/","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^NG\\\/[0-9\\.]","^NING\\\/","^PHP\\\/[0-9]","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2ip\\.ru","404enemy","7Siters","80legs","a\\.pr-cy\\.ru","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adscanner","Adstxtaggregator","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Ant\\.com","Anturis Agent","AnyEvent-HTTP\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppStoreScraperZ","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","AskQuickly","Ask Jeeves","ASPSeek","Asterias","Astute","asynchttp","Attach","autocite","Autonomy","axios\\\/","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","biz_Directory","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cg-eye","changedetection","ChangesMeter","Charlotte","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","Cosmos4j\\.feedback","Covario-IDS","Crescent","Crowsnest","Criteo","CSHttp","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Disqus\\\/","Dispatch\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconkit","faviconarchive","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedspot","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","fluffy","Flunky","flynxapp","forensiq","FoundSeoTool","http:\\\/\\\/www.neomo.de\\\/","free thumbnails","Freeuploader","Funnelback","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","github\\.com","Go [\\d\\.]* package http","Go http package","Go-Ahead-Got-It","Go-http-client","Go!Zilla","gobyus","gofetch","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-Publisher-Plugin","Google-SearchByImage","Google-Site-Verification","Google-Structured-Data-Testing-Tool","Google-Youtube-Links","google-xrawler","GoogleDocs","GoogleHC\\\/","GoogleProducer","GoogleSites","Google-Transparency-Report","Gookey","GoScraper","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hatena","Havij","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","HTTP_Compression_Test","http_request2","http_requester","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http\\.rb\\\/","http_get","HttpComponents","httphr","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","httrack","huaweisymantec","HubSpot ","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","internet_archive","Internet Ninja","InternetSeer","internetVista monitor","intraVnews","IODC","IOI","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","iZSearch","JAHHO","janforman","Jaunt\\\/","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kulturarw3","KumKie","L\\.webis","Larbin","Lavf\\\/","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lightspeedsystems","Lighthouse","Likse","Link Valet","link_thumbnailer","LinkAlarm\\\/","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreviewGenerator","LinkScan","LinksManager","LinkTiger","LinkWalker","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","looksystems\\.net","ltx71","lua-resty-http","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","mabontland","Mag-Net","MagpieRSS","Mail\\.Ru","MailChimp","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Office ","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft Data Access","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","Mister PiX","mixdata dot com","mixed-content-scan","Mixmax-LinkPreview","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Monit\\\/","monitis","Monitority\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","nagios","Najdi\\.si","Name Intelligence","Nameprotect","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","Nibbler","NICErsPRO","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-superagent","node-urllib","node\\.io","Nodemeter","NodePing","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","og-scraper","okhttp","omgili","OMSC","Online Domain Tools","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","ow\\.ly","Owler","ownCloud News","OxfordCloudService","Page Valet","page_verifier","page scorer","page2rss","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","ping\\.blo\\.gs","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","PostmanRuntime","PostPost","postrank","PowerPoint\\\/","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project 25499","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pulsepoint XT3 web scraper","Pump","Python-httplib2","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","Quora Link Preview","Qwantify","Radian6","RankActive","RankFlex","RankSonicSiteAuditor","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSOwl","safe-agent-scanner","SalesIntelligent","Saleslift","Sendsay\\.Ru","SauceNAO","SBIder","scalaj-http","scan\\.lol","ScanAlert","Scoop","scooter","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","SimpleScraper","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","search\\.thunderstone","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","SEOkicks","Seomoz","SEOprofiler","SEOsearch","seoscanners","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shrinktheweb","Sideqik","SimplePie","SimplyFast","Siphon","SISTRIX","Site-Shot\\\/","Site Sucker","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","spaziodati","SPDYCheck","Specificfeeds","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","StatusCake","Steeler","Stratagems Kumo","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","SwiteScraper","Symfony BrowserKit","Symfony2 BrowserKit","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","The Drop Reaper","The Expert HTML Source Viewer","The Knowledge AI","The Intraformant","theinternetrules","TheNomad","Thinklab","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","unchaos","unirest-java","UniversalFeedParser","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","Urlcheckr","URL Verifier","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","VidibleScraper","Virusdie","visionutils","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C_Unicorn","W3C-checklink","W3C-mobileOK","WAC-OFU","Wallpapers\\\/[0-9]+","WallpapersHD","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","web-capture\\.net","Web-sniffer","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web Sucker","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","Webthumb\\\/","WebThumbnail","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinPodder","wkhtmlto","wmtips","Woko","woorankreview","Word\\\/","WordPress\\\/","WordupinfoSearch","wotbox","WP Engine Install Performance API","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","www\\.monitor\\.us","WWWOFFLE","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-(ASR|BSC)","Y\\!J-BRW","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend_Http_Client","Zend\\\\Http\\\\Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","Zombie\\.js","Zoom\\.Mac","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)"]
\ No newline at end of file
+[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads-Creatives-Assistant","Google-Ads-Overview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
\ No newline at end of file
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt
index 1522796e..791d08bb 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt
+++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt
@@ -1,27 +1,46 @@
-.*Java.*outbrain
YLT
+^Aether
+^Amazon Simple Notification Service Agent$
+^Amazon-Route53-Health-Check-Service
^b0t$
^bluefish
^Calypso v\/
^COMODO DCV
+^Corax
^DangDang
^DavClnt
+^DHSH
+^docker\/[0-9]
+^Expanse
^FDM
^git\/
^Goose\/
^Grabber
+^Gradle\/
^HTTPClient\/
+^HTTPing
^Java\/
^Jeode\/
^Jetty\/
^Mail\/
^Mget
^Microsoft URL Control
+^Mikrotik\/
+^Netlab360
^NG\/[0-9\.]
^NING\/
-^PHP\/[0-9]
+^npm\/
+^Nuclei
+^PHP-AYMAPI\/
+^PHP\/
+^pip\/
+^pnpm\/
^RMA\/
^Ruby|Ruby\/[0-9]
+^Swurl
+^TLS tester
+^twine\/
+^ureq
^VSE\/[0-9]
^WordPress\.com
^XRL\/[0-9]
@@ -29,17 +48,18 @@
008\/
13TABS
192\.comAgent
+2GDPR\/
2ip\.ru
404enemy
7Siters
80legs
-a\.pr-cy\.ru
a3logics\.in
A6-Indexer
Abonti
Aboundex
aboutthedomain
Accoona-AI-Agent
+acebookexternalhit\/
acoon
acrylicapps\.com\/pulp
Acunetix
@@ -49,8 +69,14 @@ AddThis
ADmantX
AdminLabs
adressendeutschland
+adreview\/
adscanner
+adstxt-worker
Adstxtaggregator
+adstxt\.com
+Adyen HttpClient
+AffiliateLabz\/
+affilimate-puppeteer
agentslug
AHC
aihit
@@ -71,20 +97,23 @@ AndroidDownloadManager
Anemone
AngleSharp
annotate_google
-Ant\.com
+Anthill
Anturis Agent
+Ant\.com
AnyEvent-HTTP\/
+Apache Ant\/
Apache Droid
Apache OpenOffice
Apache-HttpAsyncClient
Apache-HttpClient
ApacheBench
Apexoo
+apimon\.de
APIs-Google
AportWorm\/
AppBeat\/
AppEngine-Google
-AppStoreScraperZ
+AppleSyndication
Aprc\/[0-9]
Arachmo
arachnode
@@ -92,21 +121,28 @@ Arachnophilia
aria2
Arukereso
asafaweb
-AskQuickly
+Asana\/
Ask Jeeves
+AskQuickly
ASPSeek
Asterias
Astute
asynchttp
Attach
+attohttpc
autocite
+AutomaticWPTester
Autonomy
+awin\.com
+AWS Security Scanner
axios\/
+a\.pr-cy\.ru
B-l-i-t-z-B-O-T
Backlink-Ceck
backlink-check
BacklinkHttpStatus
BackStreet
+BackupLand
BackWeb
Bad-Neighborhood
Badass
@@ -121,6 +157,7 @@ BBBike
BCKLINKS
BDFetch
BegunAdvertising
+Bewica-security-scan
Bidtellect
BigBozz
Bigfoot
@@ -130,11 +167,14 @@ BingPreview
binlar
biNu image cacher
Bitacle
+Bitrix link preview
biz_Directory
+BKCTwitterUnshortener\/
Black Hole
Blackboard Safeassign
BlackWidow
BlockNote\.Net
+BlogBridge
Bloglines
Bloglovin
BlogPulseLive
@@ -142,6 +182,7 @@ BlogSearch
Blogtrottr
BlowFish
boitho\.com-dc
+Boost\.Beast
BPImageWalker
Braintree-Webhooks
Branch Metrics API
@@ -169,12 +210,15 @@ catexplorador
CC Metadata Scaper
Cegbfeieh
censys
+centuryb.o.t9[at]gmail.com
Cerberian Drtrs
CERT\.at-Statistics-Survey
+cf-facebook
cg-eye
changedetection
ChangesMeter
Charlotte
+chatterino-api-cache
CheckHost
checkprivacy
CherryPicker
@@ -185,6 +229,7 @@ Chlooe
Chromaxa
CirrusExplorer
CISPA Vulnerability Notification
+CISPA Web Analyser
Citoid
CJNetworkQuality
Clarsentia
@@ -192,27 +237,34 @@ clips\.ua\.ac\.be
Cloud mapping
CloudEndure
CloudFlare-AlwaysOnline
+Cloudflare-Healthchecks
Cloudinary
cmcm\.com
coccoc
cognitiveseo
+ColdFusion
colly -
CommaFeed
Commons-HttpClient
commonscan
contactbigdatafr
contentkingapp
+Contextual Code Sites Explorer
convera
CookieReports
copyright sheriff
CopyRightCheck
Copyscape
+cortex\/
Cosmos4j\.feedback
Covario-IDS
+Craw\/
Crescent
-Crowsnest
Criteo
+Crowsnest
CSHttp
+CSSCheck
+Cula\/
curb
Curious George
curl
@@ -222,12 +274,17 @@ DAP\/NetHTTP
DareBoost
DatabaseDriverMysqli
DataCha0s
+DatadogSynthetics
Datafeedwatch
Datanyze
DataparkSearch
dataprovider
DataXu
Daum(oa)?[ \/][0-9]
+dBpoweramp
+ddline
+deeris
+delve\.ai
Demon
DeuSu
developers\.google\.com\/\+\/web\/snippet\/
@@ -237,8 +294,8 @@ Digincore
DigitalPebble
Dirbuster
Discourse Forum Onebox
-Disqus\/
Dispatch\/
+Disqus\/
DittoSpyder
dlvr
DMBrowser
@@ -246,6 +303,8 @@ DNSPod-reporting
docoloc
Dolphin http client
DomainAppender
+DomainLabz
+Domains Project\/
Donuts Content Explorer
dotMailer content retrieval
dotSemantic
@@ -258,6 +317,8 @@ drupact
Drupal \(\+http:\/\/drupal\.org\/\)
DTS Agent
dubaiindex
+DuplexWeb-Google
+DynatraceSynthetic
EARTHCOM
Easy-Thumb
EasyDL
@@ -288,20 +349,22 @@ ExactSearch
ExaleadCloudview
Excel\/
exif
+ExoRank
Exploratodo
Express WebPictures
Extreme Picture Finder
EyeNetIE
ezooms
facebookexternalhit
+facebookexternalua
facebookplatform
fairshare
Faraday v
fasthttp
Faveeo
Favicon downloader
-faviconkit
faviconarchive
+faviconkit
FavOrg
Feed Wrangler
Feedable\/
@@ -313,7 +376,9 @@ FeedBurner
feeder
Feedly
FeedshowOnline
+Feedshow\/
Feedspot
+FeedViewer\/
Feedwind\/
FeedZcollector
feeltiptop
@@ -321,6 +386,8 @@ Fetch API
Fetch\/[0-9]
Fever\/[0-9]
FHscan
+Fiery%20Feeds
+Filestack
Fimap
findlink
findthatfile
@@ -329,18 +396,24 @@ FlipboardBrowserProxy
FlipboardProxy
FlipboardRSS
Flock\/
+Florienzh\/
fluffy
Flunky
flynxapp
forensiq
+ForusP
FoundSeoTool
-http:\/\/www.neomo.de\/
+fragFINN\.de
free thumbnails
Freeuploader
+FreshRSS
+frontman
Funnelback
+Fuzz Faster U Fool
G-i-g-a-b-o-t
g00g1e\.net
ganarvisitas
+gdnplus\.com
geek-tools
Genieo
GentleSource
@@ -352,18 +425,24 @@ GetRight
getroot
GetURLInfo\/
GetWeb
+Geziyor
Ghost Inspector
GigablastOpenSource
GIS-LABS
github-camo
+GitHub-Hookshot
github\.com
-Go [\d\.]* package http
Go http package
+Go [\d\.]* package http
+Go!Zilla
Go-Ahead-Got-It
Go-http-client
-Go!Zilla
+go-mtasts\/
+gobuster
gobyus
+Gofeed
gofetch
+Goldfire Server
GomezAgent
gooblog
Goodzer\/
@@ -376,24 +455,31 @@ Google Page Speed Insights
Google PP Default
Google Search Console
Google Web Preview
+Google-Ads-Creatives-Assistant
+Google-Ads-Overview
Google-Adwords
Google-Apps-Script
Google-Calendar-Importer
Google-HotelAdsVerifier
Google-HTTP-Java-Client
+Google-InspectionTool
+Google-Podcast
Google-Publisher-Plugin
+Google-Read-Aloud
Google-SearchByImage
Google-Site-Verification
+Google-SMTP-STS
+Google-speakr
Google-Structured-Data-Testing-Tool
-Google-Youtube-Links
+Google-Transparency-Report
google-xrawler
+Google-Youtube-Links
GoogleDocs
GoogleHC\/
+GoogleProber
GoogleProducer
GoogleSites
-Google-Transparency-Report
Gookey
-GoScraper
GoSpotCheck
gosquared-thumbnailer
Gotit
@@ -419,13 +505,16 @@ Haansoft
hackney\/
Hadi Agent
HappyApps-WebCheck
+Hardenize
Hatena
Havij
+HaxerMen
HeadlessChrome
HEADMasterSEO
HeartRails_Capture
help@dataminr\.com
heritrix
+Hexometer
historious
hkedcity
hledejLevne\.cz
@@ -442,28 +531,31 @@ htdig
HTMLparser
htmlyse
HTTP Banner Detection
-HTTP_Compression_Test
-http_request2
-http_requester
http-get
HTTP-Header-Abfrage
http-kit
http-request\/
HTTP-Tiny
HTTP::Lite
-http\.rb\/
-http_get
+http:\/\/www.neomo.de\/
HttpComponents
httphr
+HTTPie
HTTPMon
httpRequest
httpscheck
httpssites_power
httpunit
HttpUrlConnection
+http\.rb\/
+HTTP_Compression_Test
+http_get
+http_request2
+http_requester
httrack
huaweisymantec
HubSpot
+HubSpot-Link-Resolver
Humanlinks
i2kconnect\/
Iblog
@@ -474,6 +566,7 @@ IDG Twitter Links Resolver
IDwhois\/
Iframely
igdeSpyder
+iGooglePortal
IlTrovatore
Image Fetch
Image Sucker
@@ -498,13 +591,15 @@ Integrity
integromedb
Intelliseek
InterGET
-internet_archive
Internet Ninja
InternetSeer
internetVista monitor
+internetwache
+internet_archive
intraVnews
IODC
IOI
+Inboxb0t
iplabel
ips-agent
IPS\/[0-9]
@@ -516,10 +611,13 @@ isitup\.org
iskanie
isUp\.li
iThemes Sync\/
+IZaBEE
iZSearch
JAHHO
janforman
Jaunt\/
+Java.*outbrain
+javelin\.io
Jbrofuzz
Jersey\/
JetCar
@@ -535,6 +633,7 @@ JolokiaPwn
Joomla
Jorgee
JS-Kit
+JungleKeyThumbnail
JustView
Kaspersky Lab CFR link resolver
Kelny\/
@@ -542,6 +641,7 @@ Kerrigan\/
KeyCDN
Keyword Density
Keywords Research
+khttp\/
KickFire
KimonoLabs\/
Kml-Google
@@ -549,11 +649,12 @@ knows\.is
KOCMOHABT
kouio
kube-probe
+kubectl
kulturarw3
KumKie
-L\.webis
Larbin
Lavf\/
+leakix\.net
LeechFTP
LeechGet
letsencrypt
@@ -564,41 +665,50 @@ Libwhisker
libwww
Licorne
Liferea\/
-Lightspeedsystems
Lighthouse
+Lightspeedsystems
Likse
+limber\.io
Link Valet
-link_thumbnailer
LinkAlarm\/
+LinkAnalyser
linkCheck
linkdex
LinkExaminer
linkfluence
linkpeek
-LinkPreviewGenerator
+LinkPreview
LinkScan
LinksManager
LinkTiger
LinkWalker
+link_thumbnailer
Lipperhey
Litemage_walker
livedoor ScreenShot
LoadImpactRload
localsearch-web
LongURL API
+longurl-r-package
+looid\.com
looksystems\.net
ltx71
lua-resty-http
+Lucee \(CFML Engine\)
+Lush Http Client
lwp-request
lwp-trivial
LWP::Simple
lycos
LYT\.SR
+L\.webis
mabontland
+MacOutlook\/
Mag-Net
MagpieRSS
-Mail\.Ru
+Mail::STS
MailChimp
+Mail\.Ru
Majestic12
makecontact\/
Mandrill
@@ -609,6 +719,8 @@ MarkWatch
Mass Downloader
masscan\/
Mata Hari
+mattermost
+Mediametric
Mediapartners-Google
mediawords
MegaIndex\.ru
@@ -619,28 +731,31 @@ Metaspinner
MetaURI
MFC_Tear_Sample
Microsearch
-Microsoft Office
+Microsoft Data Access
+Microsoft Office
Microsoft Outlook
Microsoft Windows Network Diagnostics
Microsoft-WebDAV-MiniRedir
-Microsoft Data Access
+Microsoft\.Data\.Mashup
MIDown tool
MIIxpc
Mindjet
Miniature\.io
Miniflux
+mio_httpc
+Miro-HttpClient
Mister PiX
mixdata dot com
mixed-content-scan
-Mixmax-LinkPreview
mixnode
Mnogosearch
mogimogi
Mojeek
Mojolicious \(Perl\)
-Monit\/
+Mollie
monitis
Monitority\/
+Monit\/
montastic
MonTools
Moreover
@@ -649,6 +764,7 @@ Morning Paper
MovableType
mowser
Mrcgiguy
+Mr\.4x3 Powered
MS Web Services Client Protocol
MSFrontPage
mShots
@@ -656,10 +772,13 @@ MuckRack\/
muhstik-scan
MVAClient
MxToolbox\/
+myseosnapshot
nagios
Najdi\.si
Name Intelligence
+NameFo\.com
Nameprotect
+nationalarchives
Navroad
NearSite
Needle
@@ -674,6 +793,8 @@ Netpursual
netresearch
NetShelter ContentScan
Netsparker
+NetSystemsResearch
+nettle
NetTrack
Netvibes
NetZIP
@@ -686,22 +807,26 @@ newsme
newspaper\/
Nexgate Ruby Client
NG-Search
+nghttp2
Nibbler
NICErsPRO
+NihilScio
Nikto
nineconnections
NLNZ_IAHarvester
Nmap Scripting Engine
+node-fetch
node-superagent
node-urllib
-node\.io
Nodemeter
NodePing
+node\.io
nominet\.org\.uk
nominet\.uk
Norton-Safeweb
Notifixious
notifyninja
+NotionEmbedder
nuhk
nutch
Nuzzel
@@ -709,39 +834,45 @@ nWormFeedFinder
nyawc\/
Nymesis
NYU
+Observatory\/
Ocelli\/
Octopus
oegp
Offline Explorer
Offline Navigator
-og-scraper
+OgScrper
okhttp
omgili
OMSC
Online Domain Tools
+Open Source RSS
OpenCalaisSemanticProxy
Openfind
OpenLinkProfiler
Openstat\/
OpenVAS
+OPPO A33
Optimizer
Orbiter
OrgProbe\/
orion-semantics
Outlook-Express
Outlook-iOS
-ow\.ly
Owler
+Owlin
ownCloud News
+ow\.ly
OxfordCloudService
-Page Valet
-page_verifier
page scorer
+Page Valet
page2rss
+PageFreezer
PageGrabber
PagePeeker
PageScorer
Pagespeed\/
+PageThing
+page_verifier
Panopta
panscient
Papa Foto
@@ -750,6 +881,7 @@ Pavuk
PayPal IPN
pcBrowser
Pcore-HTTP
+PDF24 URL To PDF
Pearltrees
PECL::HTTP
peerindex
@@ -759,44 +891,52 @@ Perlu -
PhantomJS Screenshoter
PhantomJS\/
Photon\/
+php-requests
phpservermon
Pi-Monster
Picscout
Picsearch
PictureFinder
Pimonster
-ping\.blo\.gs
Pingability
PingAdmin\.Ru
Pingdom
Pingoscope
PingSpot
+ping\.blo\.gs
pinterest\.com
Pixray
Pizilla
Plagger\/
+Pleroma
Ploetz \+ Zeller
Plukkie
plumanalytics
PocketImageCache
PocketParser
Pockey
+PodcastAddict\/
POE-Component-Client-HTTP
Polymail\/
Pompos
Porkbun
Port Monitor
postano
+postfix-mta-sts-resolver
PostmanRuntime
+postplanner\.com
PostPost
postrank
PowerPoint\/
+Prebid
+Prerender
Priceonomics Analysis Engine
PrintFriendly
PritTorrent
Prlog
probethenet
-Project 25499
+Project ?25499
+Project-Resonance
prospectb2b
Protopage
ProWebWalker
@@ -805,9 +945,9 @@ PRTG Network Monitor
pshtt, https scanning
PTST
PTST\/[0-9]+
-Pulsepoint XT3 web scraper
Pump
Python-httplib2
+python-httpx
python-requests
Python-urllib
Qirina Hurdler
@@ -817,12 +957,17 @@ Qseero
Qualidator
QueryN Metasearch
queuedriver
+quic-go-HTTP\/
+QuiteRSS
Quora Link Preview
Qwantify
Radian6
+RadioPublicImageResizer
+Railgun\/
RankActive
RankFlex
RankSonicSiteAuditor
+RapidLoad\/
Re-re Studio
ReactorNetty
Readability
@@ -833,6 +978,7 @@ Recorder
RecurPost\/
redback\/
ReederForMac
+Reeder\/
ReGet
RepoMonkey
request\.js
@@ -845,38 +991,44 @@ Robosourcer
Robozilla
ROI Hunter
RPT-HTTPClient
+RSSMix\/
RSSOwl
+RyowlEngine
safe-agent-scanner
SalesIntelligent
Saleslift
-Sendsay\.Ru
+SAP NetWeaver Application Server
SauceNAO
SBIder
+sc-downloader
scalaj-http
-scan\.lol
+Scamadviser-Frontend
ScanAlert
+scan\.lol
Scoop
scooter
+ScopeContentAG-HTTP-Client
ScoutJet
ScoutURLMonitor
ScrapeBox Page Scanner
-SimpleScraper
Scrapy
Screaming
ScreenShotService
Scrubby
Scrutiny\/
-search\.thunderstone
Search37
searchenginepromotionhelp
Searchestate
SearchExpress
SearchSight
+SearchWP
+search\.thunderstone
Seeker
semanticdiscovery
semanticjuice
Semiocast HTTP client
Semrush
+Sendsay\.Ru
sentry\/
SEO Browser
Seo Servis
@@ -885,11 +1037,13 @@ seo4ajax
Seobility
SEOCentro
SeoCheck
+seocompany
SEOkicks
+SEOlizer
Seomoz
SEOprofiler
-SEOsearch
seoscanners
+SEOsearch
seositecheckup
SEOstats
servernfo
@@ -900,14 +1054,16 @@ Shodan
Shoppimon
ShopWiki
ShortLinkTranslate
+shortURL lengthener
shrinktheweb
Sideqik
+Siege
SimplePie
SimplyFast
Siphon
SISTRIX
-Site-Shot\/
Site Sucker
+Site-Shot\/
Site24x7
SiteBar
Sitebeam
@@ -928,6 +1084,7 @@ Sitevigil
sitexy\.com
SkypeUriPreview
Slack\/
+sli-systems\.com
slider\.com
slurp
SlySearch
@@ -950,9 +1107,11 @@ SpaceBison
SpamExperts
Spammen
Spanner
+Spawning-AI
spaziodati
SPDYCheck
Specificfeeds
+SpeedKit
speedy
SPEng
Spinn3r
@@ -966,9 +1125,11 @@ SSL Labs
ssl-tools
StackRambler
Statastico\/
+Statically-
StatusCake
Steeler
Stratagems Kumo
+Stripe\/
Stroke\.cz
StudioFACA
StumbleUpon
@@ -978,9 +1139,11 @@ summify
SuperHTTP
Surphace Scout
Suzuran
-SwiteScraper
+swcd
Symfony BrowserKit
Symfony2 BrowserKit
+Synapse\/
+Syndirella\/
SynHttpClient-Built
Sysomos
sysscan
@@ -990,6 +1153,8 @@ tAkeOut
Tarantula\/
Taringa UGC
TarmotGezgin
+tchelebi\.io
+techiaith\.cymru
Teleport
Telesoft
Telesphoreo
@@ -1000,13 +1165,15 @@ terrainformatica
Test Certificate Info
testuri
Tetrahedron
+TextRazor Downloader
The Drop Reaper
The Expert HTML Source Viewer
-The Knowledge AI
The Intraformant
+The Knowledge AI
theinternetrules
TheNomad
Thinklab
+Thumbor
Thumbshots
ThumbSniper
timewe\.net
@@ -1040,9 +1207,11 @@ Typhoeus
ubermetrics-technologies
uclassify
UdmSearch
+ultimate_sitemap_parser
unchaos
unirest-java
UniversalFeedParser
+unshortenit
Unshorten\.It
Untiny
UnwindFetchor
@@ -1050,8 +1219,8 @@ updated
updown\.io daemon
Upflow
Uptimia
-Urlcheckr
URL Verifier
+Urlcheckr
URLitor
urlresolver
Urlstat
@@ -1059,15 +1228,16 @@ URLTester
UrlTrends Ranking Updater
URLy Warning
URLy\.Warning
+URL\/Emacs
Vacuum
Vagabondo
VB Project
vBSEO
VCI
via ggpht\.com GoogleImageProxy
-VidibleScraper
Virusdie
visionutils
+Visual Rights Group
vkShare
VoidEYE
Voil
@@ -1078,19 +1248,18 @@ VSB-TUO\/
Vulnbusters Meter
VYU2
w3af\.org
-W3C_Unicorn
W3C-checklink
W3C-mobileOK
+W3C_Unicorn
WAC-OFU
-Wallpapers\/[0-9]+
+WakeletLinkExpander
WallpapersHD
+Wallpapers\/[0-9]+
wangling
Wappalyzer
WatchMouse
WbSrch\/
WDT\.io
-web-capture\.net
-Web-sniffer
Web Auto
Web Collage
Web Enhancer
@@ -1098,7 +1267,10 @@ Web Fetch
Web Fuck
Web Pix
Web Sauger
+Web spyder
Web Sucker
+web-capture\.net
+Web-sniffer
Webalta
Webauskunft
WebAuto
@@ -1136,8 +1308,9 @@ WebSniffer
Webster
WebStripper
WebSucker
-Webthumb\/
+webtech\/
WebThumbnail
+Webthumb\/
WebWhacker
WebZIP
WeLikeLinks
@@ -1146,24 +1319,31 @@ WeSEE
wf84
Wfuzz\/
wget
+WhatCMS
WhatsApp
WhatsMyIP
WhatWeb
WhereGoes\?
Whibse
+WhoAPI\/
WhoRunsCoinHive
Whynder Magnet
Windows-RSS-Platform
+WinHttp-Autoproxy-Service
+WinHTTP\/
WinPodder
wkhtmlto
wmtips
Woko
+Wolfram HTTPClient
woorankreview
-Word\/
WordPress\/
WordupinfoSearch
+Word\/
+worldping-api
wotbox
WP Engine Install Performance API
+WP Rocket
wpif
wprecon\.com survey
WPScan
@@ -1173,20 +1353,21 @@ WWW-Collector-E
WWW-Mechanize
WWW::Document
WWW::Mechanize
-www\.monitor\.us
WWWOFFLE
+www\.monitor\.us
x09Mozilla
x22Mozilla
XaxisSemanticsClassifier
+XenForo\/
Xenu Link Sleuth
XING-contenttabreceiver
xpymep([0-9]?)\.exe
-Y!J-(ASR|BSC)
-Y\!J-BRW
+Y!J-[A-Z][A-Z][A-Z]
Yaanb
yacy
Yahoo Link Preview
YahooCacheSystem
+YahooMailProxy
YahooYSMcm
YandeG
Yandex(?!Search)
@@ -1194,6 +1375,7 @@ yanga
yeti
Yo-yo
Yoleo Consumer
+yomins\.com
yoogliFetchAgent
YottaaMonitor
Your-Website-Sucks
@@ -1205,13 +1387,15 @@ Zade
Zao
Zauba
Zemanta Aggregator
-Zend_Http_Client
Zend\\Http\\Client
+Zend_Http_Client
Zermelo
Zeus
zgrab
ZnajdzFoto
+ZnHTTP
Zombie\.js
Zoom\.Mac
+ZoteroTranslationServer
ZyBorg
-[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)
\ No newline at end of file
+[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)
\ No newline at end of file
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json
index a18eb985..e7e01416 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json
+++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json
@@ -1 +1 @@
-["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"]
\ No newline at end of file
+["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"]
\ No newline at end of file
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt
index da56db9b..a44a99cb 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt
+++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt
@@ -45,4 +45,6 @@ cubot
; IDbot
; ID bot
; POWER BOT
-;
\ No newline at end of file
+OCTOPUS-CORE
+htc_botdugls
+super\/\d+\/Android\/\d+
\ No newline at end of file
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php b/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php
index 1067976b..3ea284a7 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php
+++ b/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php
@@ -20,9 +20,9 @@ class CrawlerDetect
/**
* The user agent.
*
- * @var null
+ * @var string|null
*/
- protected $userAgent = null;
+ protected $userAgent;
/**
* Headers that contain a user agent.
@@ -93,7 +93,7 @@ class CrawlerDetect
* Compile the regex patterns into one regex string.
*
* @param array
- *
+ *
* @return string
*/
public function compileRegex($patterns)
@@ -138,7 +138,7 @@ class CrawlerDetect
/**
* Set the user agent.
*
- * @param string $userAgent
+ * @param string|null $userAgent
*/
public function setUserAgent($userAgent)
{
@@ -165,20 +165,14 @@ class CrawlerDetect
$agent = trim(preg_replace(
"/{$this->compiledExclusions}/i",
'',
- $userAgent ?: $this->userAgent
+ $userAgent ?: $this->userAgent ?: ''
));
- if ($agent == '') {
+ if ($agent === '') {
return false;
}
- $result = preg_match("/{$this->compiledRegex}/i", $agent, $matches);
-
- if ($matches) {
- $this->matches = $matches;
- }
-
- return (bool) $result;
+ return (bool) preg_match("/{$this->compiledRegex}/i", $agent, $this->matches);
}
/**
@@ -190,4 +184,13 @@ class CrawlerDetect
{
return isset($this->matches[0]) ? $this->matches[0] : null;
}
+
+
+ /**
+ * @return string|null
+ */
+ public function getUserAgent()
+ {
+ return $this->userAgent;
+ }
}
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php
index 26ea8e5f..ffe10f51 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php
+++ b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php
@@ -15,14 +15,14 @@ abstract class AbstractProvider
{
/**
* The data set.
- *
+ *
* @var array
*/
protected $data;
/**
* Return the data set.
- *
+ *
* @return array
*/
public function getAll()
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php
index a9070565..a44edd23 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php
+++ b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php
@@ -19,30 +19,49 @@ class Crawlers extends AbstractProvider
* @var array
*/
protected $data = array(
- '.*Java.*outbrain',
' YLT',
+ '^Aether',
+ '^Amazon Simple Notification Service Agent$',
+ '^Amazon-Route53-Health-Check-Service',
'^b0t$',
'^bluefish ',
'^Calypso v\/',
'^COMODO DCV',
+ '^Corax',
'^DangDang',
'^DavClnt',
+ '^DHSH',
+ '^docker\/[0-9]',
+ '^Expanse',
'^FDM ',
'^git\/',
'^Goose\/',
'^Grabber',
+ '^Gradle\/',
'^HTTPClient\/',
+ '^HTTPing',
'^Java\/',
'^Jeode\/',
'^Jetty\/',
'^Mail\/',
'^Mget',
'^Microsoft URL Control',
+ '^Mikrotik\/',
+ '^Netlab360',
'^NG\/[0-9\.]',
'^NING\/',
- '^PHP\/[0-9]',
+ '^npm\/',
+ '^Nuclei',
+ '^PHP-AYMAPI\/',
+ '^PHP\/',
+ '^pip\/',
+ '^pnpm\/',
'^RMA\/',
'^Ruby|Ruby\/[0-9]',
+ '^Swurl ',
+ '^TLS tester ',
+ '^twine\/',
+ '^ureq',
'^VSE\/[0-9]',
'^WordPress\.com',
'^XRL\/[0-9]',
@@ -50,17 +69,18 @@ class Crawlers extends AbstractProvider
'008\/',
'13TABS',
'192\.comAgent',
+ '2GDPR\/',
'2ip\.ru',
'404enemy',
'7Siters',
'80legs',
- 'a\.pr-cy\.ru',
'a3logics\.in',
'A6-Indexer',
'Abonti',
'Aboundex',
'aboutthedomain',
'Accoona-AI-Agent',
+ 'acebookexternalhit\/',
'acoon',
'acrylicapps\.com\/pulp',
'Acunetix',
@@ -70,8 +90,14 @@ class Crawlers extends AbstractProvider
'ADmantX',
'AdminLabs',
'adressendeutschland',
+ 'adreview\/',
'adscanner',
+ 'adstxt-worker',
'Adstxtaggregator',
+ 'adstxt\.com',
+ 'Adyen HttpClient',
+ 'AffiliateLabz\/',
+ 'affilimate-puppeteer',
'agentslug',
'AHC',
'aihit',
@@ -92,20 +118,23 @@ class Crawlers extends AbstractProvider
'Anemone',
'AngleSharp',
'annotate_google',
- 'Ant\.com',
+ 'Anthill',
'Anturis Agent',
+ 'Ant\.com',
'AnyEvent-HTTP\/',
+ 'Apache Ant\/',
'Apache Droid',
'Apache OpenOffice',
'Apache-HttpAsyncClient',
'Apache-HttpClient',
'ApacheBench',
'Apexoo',
+ 'apimon\.de',
'APIs-Google',
'AportWorm\/',
'AppBeat\/',
'AppEngine-Google',
- 'AppStoreScraperZ',
+ 'AppleSyndication',
'Aprc\/[0-9]',
'Arachmo',
'arachnode',
@@ -113,21 +142,28 @@ class Crawlers extends AbstractProvider
'aria2',
'Arukereso',
'asafaweb',
- 'AskQuickly',
+ 'Asana\/',
'Ask Jeeves',
+ 'AskQuickly',
'ASPSeek',
'Asterias',
'Astute',
'asynchttp',
'Attach',
+ 'attohttpc',
'autocite',
+ 'AutomaticWPTester',
'Autonomy',
+ 'awin\.com',
+ 'AWS Security Scanner',
'axios\/',
+ 'a\.pr-cy\.ru',
'B-l-i-t-z-B-O-T',
'Backlink-Ceck',
'backlink-check',
'BacklinkHttpStatus',
'BackStreet',
+ 'BackupLand',
'BackWeb',
'Bad-Neighborhood',
'Badass',
@@ -142,6 +178,7 @@ class Crawlers extends AbstractProvider
'BCKLINKS',
'BDFetch',
'BegunAdvertising',
+ 'Bewica-security-scan',
'Bidtellect',
'BigBozz',
'Bigfoot',
@@ -151,11 +188,14 @@ class Crawlers extends AbstractProvider
'binlar',
'biNu image cacher',
'Bitacle',
+ 'Bitrix link preview',
'biz_Directory',
+ 'BKCTwitterUnshortener\/',
'Black Hole',
'Blackboard Safeassign',
'BlackWidow',
'BlockNote\.Net',
+ 'BlogBridge',
'Bloglines',
'Bloglovin',
'BlogPulseLive',
@@ -163,6 +203,7 @@ class Crawlers extends AbstractProvider
'Blogtrottr',
'BlowFish',
'boitho\.com-dc',
+ 'Boost\.Beast',
'BPImageWalker',
'Braintree-Webhooks',
'Branch Metrics API',
@@ -190,12 +231,15 @@ class Crawlers extends AbstractProvider
'CC Metadata Scaper',
'Cegbfeieh',
'censys',
+ 'centuryb.o.t9[at]gmail.com',
'Cerberian Drtrs',
'CERT\.at-Statistics-Survey',
+ 'cf-facebook',
'cg-eye',
'changedetection',
'ChangesMeter',
'Charlotte',
+ 'chatterino-api-cache',
'CheckHost',
'checkprivacy',
'CherryPicker',
@@ -206,6 +250,7 @@ class Crawlers extends AbstractProvider
'Chromaxa',
'CirrusExplorer',
'CISPA Vulnerability Notification',
+ 'CISPA Web Analyser',
'Citoid',
'CJNetworkQuality',
'Clarsentia',
@@ -213,27 +258,34 @@ class Crawlers extends AbstractProvider
'Cloud mapping',
'CloudEndure',
'CloudFlare-AlwaysOnline',
+ 'Cloudflare-Healthchecks',
'Cloudinary',
'cmcm\.com',
'coccoc',
'cognitiveseo',
+ 'ColdFusion',
'colly -',
'CommaFeed',
'Commons-HttpClient',
'commonscan',
'contactbigdatafr',
'contentkingapp',
+ 'Contextual Code Sites Explorer',
'convera',
'CookieReports',
'copyright sheriff',
'CopyRightCheck',
'Copyscape',
+ 'cortex\/',
'Cosmos4j\.feedback',
'Covario-IDS',
+ 'Craw\/',
'Crescent',
- 'Crowsnest',
'Criteo',
+ 'Crowsnest',
'CSHttp',
+ 'CSSCheck',
+ 'Cula\/',
'curb',
'Curious George',
'curl',
@@ -243,12 +295,17 @@ class Crawlers extends AbstractProvider
'DareBoost',
'DatabaseDriverMysqli',
'DataCha0s',
+ 'DatadogSynthetics',
'Datafeedwatch',
'Datanyze',
'DataparkSearch',
'dataprovider',
'DataXu',
'Daum(oa)?[ \/][0-9]',
+ 'dBpoweramp',
+ 'ddline',
+ 'deeris',
+ 'delve\.ai',
'Demon',
'DeuSu',
'developers\.google\.com\/\+\/web\/snippet\/',
@@ -258,8 +315,8 @@ class Crawlers extends AbstractProvider
'DigitalPebble',
'Dirbuster',
'Discourse Forum Onebox',
- 'Disqus\/',
'Dispatch\/',
+ 'Disqus\/',
'DittoSpyder',
'dlvr',
'DMBrowser',
@@ -267,6 +324,8 @@ class Crawlers extends AbstractProvider
'docoloc',
'Dolphin http client',
'DomainAppender',
+ 'DomainLabz',
+ 'Domains Project\/',
'Donuts Content Explorer',
'dotMailer content retrieval',
'dotSemantic',
@@ -279,6 +338,8 @@ class Crawlers extends AbstractProvider
'Drupal \(\+http:\/\/drupal\.org\/\)',
'DTS Agent',
'dubaiindex',
+ 'DuplexWeb-Google',
+ 'DynatraceSynthetic',
'EARTHCOM',
'Easy-Thumb',
'EasyDL',
@@ -309,20 +370,22 @@ class Crawlers extends AbstractProvider
'ExaleadCloudview',
'Excel\/',
'exif',
+ 'ExoRank',
'Exploratodo',
'Express WebPictures',
'Extreme Picture Finder',
'EyeNetIE',
'ezooms',
'facebookexternalhit',
+ 'facebookexternalua',
'facebookplatform',
'fairshare',
'Faraday v',
'fasthttp',
'Faveeo',
'Favicon downloader',
- 'faviconkit',
'faviconarchive',
+ 'faviconkit',
'FavOrg',
'Feed Wrangler',
'Feedable\/',
@@ -334,7 +397,9 @@ class Crawlers extends AbstractProvider
'feeder',
'Feedly',
'FeedshowOnline',
+ 'Feedshow\/',
'Feedspot',
+ 'FeedViewer\/',
'Feedwind\/',
'FeedZcollector',
'feeltiptop',
@@ -342,6 +407,8 @@ class Crawlers extends AbstractProvider
'Fetch\/[0-9]',
'Fever\/[0-9]',
'FHscan',
+ 'Fiery%20Feeds',
+ 'Filestack',
'Fimap',
'findlink',
'findthatfile',
@@ -350,18 +417,24 @@ class Crawlers extends AbstractProvider
'FlipboardProxy',
'FlipboardRSS',
'Flock\/',
+ 'Florienzh\/',
'fluffy',
'Flunky',
'flynxapp',
'forensiq',
+ 'ForusP',
'FoundSeoTool',
- 'http:\/\/www.neomo.de\/', //'Francis [Bot]'
+ 'fragFINN\.de',
'free thumbnails',
'Freeuploader',
+ 'FreshRSS',
+ 'frontman',
'Funnelback',
+ 'Fuzz Faster U Fool',
'G-i-g-a-b-o-t',
'g00g1e\.net',
'ganarvisitas',
+ 'gdnplus\.com',
'geek-tools',
'Genieo',
'GentleSource',
@@ -373,18 +446,24 @@ class Crawlers extends AbstractProvider
'getroot',
'GetURLInfo\/',
'GetWeb',
+ 'Geziyor',
'Ghost Inspector',
'GigablastOpenSource',
'GIS-LABS',
'github-camo',
+ 'GitHub-Hookshot',
'github\.com',
- 'Go [\d\.]* package http',
'Go http package',
+ 'Go [\d\.]* package http',
+ 'Go!Zilla',
'Go-Ahead-Got-It',
'Go-http-client',
- 'Go!Zilla',
+ 'go-mtasts\/',
+ 'gobuster',
'gobyus',
+ 'Gofeed',
'gofetch',
+ 'Goldfire Server',
'GomezAgent',
'gooblog',
'Goodzer\/',
@@ -397,24 +476,31 @@ class Crawlers extends AbstractProvider
'Google PP Default',
'Google Search Console',
'Google Web Preview',
+ 'Google-Ads-Creatives-Assistant',
+ 'Google-Ads-Overview',
'Google-Adwords',
'Google-Apps-Script',
'Google-Calendar-Importer',
'Google-HotelAdsVerifier',
'Google-HTTP-Java-Client',
+ 'Google-InspectionTool',
+ 'Google-Podcast',
'Google-Publisher-Plugin',
+ 'Google-Read-Aloud',
'Google-SearchByImage',
'Google-Site-Verification',
+ 'Google-SMTP-STS',
+ 'Google-speakr',
'Google-Structured-Data-Testing-Tool',
- 'Google-Youtube-Links',
+ 'Google-Transparency-Report',
'google-xrawler',
+ 'Google-Youtube-Links',
'GoogleDocs',
'GoogleHC\/',
+ 'GoogleProber',
'GoogleProducer',
'GoogleSites',
- 'Google-Transparency-Report',
'Gookey',
- 'GoScraper',
'GoSpotCheck',
'gosquared-thumbnailer',
'Gotit',
@@ -440,13 +526,16 @@ class Crawlers extends AbstractProvider
'hackney\/',
'Hadi Agent',
'HappyApps-WebCheck',
+ 'Hardenize',
'Hatena',
'Havij',
+ 'HaxerMen',
'HeadlessChrome',
'HEADMasterSEO',
'HeartRails_Capture',
'help@dataminr\.com',
'heritrix',
+ 'Hexometer',
'historious',
'hkedcity',
'hledejLevne\.cz',
@@ -463,28 +552,31 @@ class Crawlers extends AbstractProvider
'HTMLparser',
'htmlyse',
'HTTP Banner Detection',
- 'HTTP_Compression_Test',
- 'http_request2',
- 'http_requester',
'http-get',
'HTTP-Header-Abfrage',
'http-kit',
'http-request\/',
'HTTP-Tiny',
'HTTP::Lite',
- 'http\.rb\/',
- 'http_get',
+ 'http:\/\/www.neomo.de\/', //'Francis [Bot]'
'HttpComponents',
'httphr',
+ 'HTTPie',
'HTTPMon',
'httpRequest',
'httpscheck',
'httpssites_power',
'httpunit',
'HttpUrlConnection',
+ 'http\.rb\/',
+ 'HTTP_Compression_Test',
+ 'http_get',
+ 'http_request2',
+ 'http_requester',
'httrack',
'huaweisymantec',
'HubSpot ',
+ 'HubSpot-Link-Resolver',
'Humanlinks',
'i2kconnect\/',
'Iblog',
@@ -495,6 +587,7 @@ class Crawlers extends AbstractProvider
'IDwhois\/',
'Iframely',
'igdeSpyder',
+ 'iGooglePortal',
'IlTrovatore',
'Image Fetch',
'Image Sucker',
@@ -519,13 +612,15 @@ class Crawlers extends AbstractProvider
'integromedb',
'Intelliseek',
'InterGET',
- 'internet_archive',
'Internet Ninja',
'InternetSeer',
'internetVista monitor',
+ 'internetwache',
+ 'internet_archive',
'intraVnews',
'IODC',
'IOI',
+ 'Inboxb0t',
'iplabel',
'ips-agent',
'IPS\/[0-9]',
@@ -537,10 +632,13 @@ class Crawlers extends AbstractProvider
'iskanie',
'isUp\.li',
'iThemes Sync\/',
+ 'IZaBEE',
'iZSearch',
'JAHHO',
'janforman',
'Jaunt\/',
+ 'Java.*outbrain',
+ 'javelin\.io',
'Jbrofuzz',
'Jersey\/',
'JetCar',
@@ -556,6 +654,7 @@ class Crawlers extends AbstractProvider
'Joomla',
'Jorgee',
'JS-Kit',
+ 'JungleKeyThumbnail',
'JustView',
'Kaspersky Lab CFR link resolver',
'Kelny\/',
@@ -563,6 +662,7 @@ class Crawlers extends AbstractProvider
'KeyCDN',
'Keyword Density',
'Keywords Research',
+ 'khttp\/',
'KickFire',
'KimonoLabs\/',
'Kml-Google',
@@ -570,11 +670,12 @@ class Crawlers extends AbstractProvider
'KOCMOHABT',
'kouio',
'kube-probe',
+ 'kubectl',
'kulturarw3',
'KumKie',
- 'L\.webis',
'Larbin',
'Lavf\/',
+ 'leakix\.net',
'LeechFTP',
'LeechGet',
'letsencrypt',
@@ -585,41 +686,50 @@ class Crawlers extends AbstractProvider
'libwww',
'Licorne',
'Liferea\/',
- 'Lightspeedsystems',
'Lighthouse',
+ 'Lightspeedsystems',
'Likse',
+ 'limber\.io',
'Link Valet',
- 'link_thumbnailer',
'LinkAlarm\/',
+ 'LinkAnalyser',
'linkCheck',
'linkdex',
'LinkExaminer',
'linkfluence',
'linkpeek',
- 'LinkPreviewGenerator',
+ 'LinkPreview',
'LinkScan',
'LinksManager',
'LinkTiger',
'LinkWalker',
+ 'link_thumbnailer',
'Lipperhey',
'Litemage_walker',
'livedoor ScreenShot',
'LoadImpactRload',
'localsearch-web',
'LongURL API',
+ 'longurl-r-package',
+ 'looid\.com',
'looksystems\.net',
'ltx71',
'lua-resty-http',
+ 'Lucee \(CFML Engine\)',
+ 'Lush Http Client',
'lwp-request',
'lwp-trivial',
'LWP::Simple',
'lycos',
'LYT\.SR',
+ 'L\.webis',
'mabontland',
+ 'MacOutlook\/',
'Mag-Net',
'MagpieRSS',
- 'Mail\.Ru',
+ 'Mail::STS',
'MailChimp',
+ 'Mail\.Ru',
'Majestic12',
'makecontact\/',
'Mandrill',
@@ -630,6 +740,8 @@ class Crawlers extends AbstractProvider
'Mass Downloader',
'masscan\/',
'Mata Hari',
+ 'mattermost',
+ 'Mediametric',
'Mediapartners-Google',
'mediawords',
'MegaIndex\.ru',
@@ -640,28 +752,31 @@ class Crawlers extends AbstractProvider
'MetaURI',
'MFC_Tear_Sample',
'Microsearch',
- 'Microsoft Office ',
+ 'Microsoft Data Access',
+ 'Microsoft Office',
'Microsoft Outlook',
'Microsoft Windows Network Diagnostics',
'Microsoft-WebDAV-MiniRedir',
- 'Microsoft Data Access',
+ 'Microsoft\.Data\.Mashup',
'MIDown tool',
'MIIxpc',
'Mindjet',
'Miniature\.io',
'Miniflux',
+ 'mio_httpc',
+ 'Miro-HttpClient',
'Mister PiX',
'mixdata dot com',
'mixed-content-scan',
- 'Mixmax-LinkPreview',
'mixnode',
'Mnogosearch',
'mogimogi',
'Mojeek',
'Mojolicious \(Perl\)',
- 'Monit\/',
+ 'Mollie',
'monitis',
'Monitority\/',
+ 'Monit\/',
'montastic',
'MonTools',
'Moreover',
@@ -670,6 +785,7 @@ class Crawlers extends AbstractProvider
'MovableType',
'mowser',
'Mrcgiguy',
+ 'Mr\.4x3 Powered',
'MS Web Services Client Protocol',
'MSFrontPage',
'mShots',
@@ -677,10 +793,13 @@ class Crawlers extends AbstractProvider
'muhstik-scan',
'MVAClient',
'MxToolbox\/',
+ 'myseosnapshot',
'nagios',
'Najdi\.si',
'Name Intelligence',
+ 'NameFo\.com',
'Nameprotect',
+ 'nationalarchives',
'Navroad',
'NearSite',
'Needle',
@@ -695,6 +814,8 @@ class Crawlers extends AbstractProvider
'netresearch',
'NetShelter ContentScan',
'Netsparker',
+ 'NetSystemsResearch',
+ 'nettle',
'NetTrack',
'Netvibes',
'NetZIP',
@@ -707,22 +828,26 @@ class Crawlers extends AbstractProvider
'newspaper\/',
'Nexgate Ruby Client',
'NG-Search',
+ 'nghttp2',
'Nibbler',
'NICErsPRO',
+ 'NihilScio',
'Nikto',
'nineconnections',
'NLNZ_IAHarvester',
'Nmap Scripting Engine',
+ 'node-fetch',
'node-superagent',
'node-urllib',
- 'node\.io',
'Nodemeter',
'NodePing',
+ 'node\.io',
'nominet\.org\.uk',
'nominet\.uk',
'Norton-Safeweb',
'Notifixious',
'notifyninja',
+ 'NotionEmbedder',
'nuhk',
'nutch',
'Nuzzel',
@@ -730,39 +855,45 @@ class Crawlers extends AbstractProvider
'nyawc\/',
'Nymesis',
'NYU',
+ 'Observatory\/',
'Ocelli\/',
'Octopus',
'oegp',
'Offline Explorer',
'Offline Navigator',
- 'og-scraper',
+ 'OgScrper',
'okhttp',
'omgili',
'OMSC',
'Online Domain Tools',
+ 'Open Source RSS',
'OpenCalaisSemanticProxy',
'Openfind',
'OpenLinkProfiler',
'Openstat\/',
'OpenVAS',
+ 'OPPO A33',
'Optimizer',
'Orbiter',
'OrgProbe\/',
'orion-semantics',
'Outlook-Express',
'Outlook-iOS',
- 'ow\.ly',
'Owler',
+ 'Owlin',
'ownCloud News',
+ 'ow\.ly',
'OxfordCloudService',
- 'Page Valet',
- 'page_verifier',
'page scorer',
+ 'Page Valet',
'page2rss',
+ 'PageFreezer',
'PageGrabber',
'PagePeeker',
'PageScorer',
'Pagespeed\/',
+ 'PageThing',
+ 'page_verifier',
'Panopta',
'panscient',
'Papa Foto',
@@ -771,6 +902,7 @@ class Crawlers extends AbstractProvider
'PayPal IPN',
'pcBrowser',
'Pcore-HTTP',
+ 'PDF24 URL To PDF',
'Pearltrees',
'PECL::HTTP',
'peerindex',
@@ -780,44 +912,52 @@ class Crawlers extends AbstractProvider
'PhantomJS Screenshoter',
'PhantomJS\/',
'Photon\/',
+ 'php-requests',
'phpservermon',
'Pi-Monster',
'Picscout',
'Picsearch',
'PictureFinder',
'Pimonster',
- 'ping\.blo\.gs',
'Pingability',
'PingAdmin\.Ru',
'Pingdom',
'Pingoscope',
'PingSpot',
+ 'ping\.blo\.gs',
'pinterest\.com',
'Pixray',
'Pizilla',
'Plagger\/',
+ 'Pleroma ',
'Ploetz \+ Zeller',
'Plukkie',
'plumanalytics',
'PocketImageCache',
'PocketParser',
'Pockey',
+ 'PodcastAddict\/',
'POE-Component-Client-HTTP',
'Polymail\/',
'Pompos',
'Porkbun',
'Port Monitor',
'postano',
+ 'postfix-mta-sts-resolver',
'PostmanRuntime',
+ 'postplanner\.com',
'PostPost',
'postrank',
'PowerPoint\/',
+ 'Prebid',
+ 'Prerender',
'Priceonomics Analysis Engine',
'PrintFriendly',
'PritTorrent',
'Prlog',
'probethenet',
- 'Project 25499',
+ 'Project ?25499',
+ 'Project-Resonance',
'prospectb2b',
'Protopage',
'ProWebWalker',
@@ -826,9 +966,9 @@ class Crawlers extends AbstractProvider
'pshtt, https scanning',
'PTST ',
'PTST\/[0-9]+',
- 'Pulsepoint XT3 web scraper',
'Pump',
'Python-httplib2',
+ 'python-httpx',
'python-requests',
'Python-urllib',
'Qirina Hurdler',
@@ -838,12 +978,17 @@ class Crawlers extends AbstractProvider
'Qualidator',
'QueryN Metasearch',
'queuedriver',
+ 'quic-go-HTTP\/',
+ 'QuiteRSS',
'Quora Link Preview',
'Qwantify',
'Radian6',
+ 'RadioPublicImageResizer',
+ 'Railgun\/',
'RankActive',
'RankFlex',
'RankSonicSiteAuditor',
+ 'RapidLoad\/',
'Re-re Studio',
'ReactorNetty',
'Readability',
@@ -854,6 +999,7 @@ class Crawlers extends AbstractProvider
'RecurPost\/',
'redback\/',
'ReederForMac',
+ 'Reeder\/',
'ReGet',
'RepoMonkey',
'request\.js',
@@ -866,38 +1012,44 @@ class Crawlers extends AbstractProvider
'Robozilla',
'ROI Hunter',
'RPT-HTTPClient',
+ 'RSSMix\/',
'RSSOwl',
+ 'RyowlEngine',
'safe-agent-scanner',
'SalesIntelligent',
'Saleslift',
- 'Sendsay\.Ru',
+ 'SAP NetWeaver Application Server',
'SauceNAO',
'SBIder',
+ 'sc-downloader',
'scalaj-http',
- 'scan\.lol',
+ 'Scamadviser-Frontend',
'ScanAlert',
+ 'scan\.lol',
'Scoop',
'scooter',
+ 'ScopeContentAG-HTTP-Client',
'ScoutJet',
'ScoutURLMonitor',
'ScrapeBox Page Scanner',
- 'SimpleScraper',
'Scrapy',
'Screaming',
'ScreenShotService',
'Scrubby',
'Scrutiny\/',
- 'search\.thunderstone',
'Search37',
'searchenginepromotionhelp',
'Searchestate',
'SearchExpress',
'SearchSight',
+ 'SearchWP',
+ 'search\.thunderstone',
'Seeker',
'semanticdiscovery',
'semanticjuice',
'Semiocast HTTP client',
'Semrush',
+ 'Sendsay\.Ru',
'sentry\/',
'SEO Browser',
'Seo Servis',
@@ -906,11 +1058,13 @@ class Crawlers extends AbstractProvider
'Seobility',
'SEOCentro',
'SeoCheck',
+ 'seocompany',
'SEOkicks',
+ 'SEOlizer',
'Seomoz',
'SEOprofiler',
- 'SEOsearch',
'seoscanners',
+ 'SEOsearch',
'seositecheckup',
'SEOstats',
'servernfo',
@@ -921,14 +1075,16 @@ class Crawlers extends AbstractProvider
'Shoppimon',
'ShopWiki',
'ShortLinkTranslate',
+ 'shortURL lengthener',
'shrinktheweb',
'Sideqik',
+ 'Siege',
'SimplePie',
'SimplyFast',
'Siphon',
'SISTRIX',
- 'Site-Shot\/',
'Site Sucker',
+ 'Site-Shot\/',
'Site24x7',
'SiteBar',
'Sitebeam',
@@ -949,6 +1105,7 @@ class Crawlers extends AbstractProvider
'sitexy\.com',
'SkypeUriPreview',
'Slack\/',
+ 'sli-systems\.com',
'slider\.com',
'slurp',
'SlySearch',
@@ -971,9 +1128,11 @@ class Crawlers extends AbstractProvider
'SpamExperts',
'Spammen',
'Spanner',
+ 'Spawning-AI',
'spaziodati',
'SPDYCheck',
'Specificfeeds',
+ 'SpeedKit',
'speedy',
'SPEng',
'Spinn3r',
@@ -987,9 +1146,11 @@ class Crawlers extends AbstractProvider
'ssl-tools',
'StackRambler',
'Statastico\/',
+ 'Statically-',
'StatusCake',
'Steeler',
'Stratagems Kumo',
+ 'Stripe\/',
'Stroke\.cz',
'StudioFACA',
'StumbleUpon',
@@ -999,9 +1160,11 @@ class Crawlers extends AbstractProvider
'SuperHTTP',
'Surphace Scout',
'Suzuran',
- 'SwiteScraper',
+ 'swcd ',
'Symfony BrowserKit',
'Symfony2 BrowserKit',
+ 'Synapse\/',
+ 'Syndirella\/',
'SynHttpClient-Built',
'Sysomos',
'sysscan',
@@ -1011,6 +1174,8 @@ class Crawlers extends AbstractProvider
'Tarantula\/',
'Taringa UGC',
'TarmotGezgin',
+ 'tchelebi\.io',
+ 'techiaith\.cymru',
'Teleport',
'Telesoft',
'Telesphoreo',
@@ -1021,13 +1186,15 @@ class Crawlers extends AbstractProvider
'Test Certificate Info',
'testuri',
'Tetrahedron',
+ 'TextRazor Downloader',
'The Drop Reaper',
'The Expert HTML Source Viewer',
- 'The Knowledge AI',
'The Intraformant',
+ 'The Knowledge AI',
'theinternetrules',
'TheNomad',
'Thinklab',
+ 'Thumbor',
'Thumbshots',
'ThumbSniper',
'timewe\.net',
@@ -1061,9 +1228,11 @@ class Crawlers extends AbstractProvider
'ubermetrics-technologies',
'uclassify',
'UdmSearch',
+ 'ultimate_sitemap_parser',
'unchaos',
'unirest-java',
'UniversalFeedParser',
+ 'unshortenit',
'Unshorten\.It',
'Untiny',
'UnwindFetchor',
@@ -1071,8 +1240,8 @@ class Crawlers extends AbstractProvider
'updown\.io daemon',
'Upflow',
'Uptimia',
- 'Urlcheckr',
'URL Verifier',
+ 'Urlcheckr',
'URLitor',
'urlresolver',
'Urlstat',
@@ -1080,15 +1249,16 @@ class Crawlers extends AbstractProvider
'UrlTrends Ranking Updater',
'URLy Warning',
'URLy\.Warning',
+ 'URL\/Emacs',
'Vacuum',
'Vagabondo',
'VB Project',
'vBSEO',
'VCI',
'via ggpht\.com GoogleImageProxy',
- 'VidibleScraper',
'Virusdie',
'visionutils',
+ 'Visual Rights Group',
'vkShare',
'VoidEYE',
'Voil',
@@ -1099,19 +1269,18 @@ class Crawlers extends AbstractProvider
'Vulnbusters Meter',
'VYU2',
'w3af\.org',
- 'W3C_Unicorn',
'W3C-checklink',
'W3C-mobileOK',
+ 'W3C_Unicorn',
'WAC-OFU',
- 'Wallpapers\/[0-9]+',
+ 'WakeletLinkExpander',
'WallpapersHD',
+ 'Wallpapers\/[0-9]+',
'wangling',
'Wappalyzer',
'WatchMouse',
'WbSrch\/',
'WDT\.io',
- 'web-capture\.net',
- 'Web-sniffer',
'Web Auto',
'Web Collage',
'Web Enhancer',
@@ -1119,7 +1288,10 @@ class Crawlers extends AbstractProvider
'Web Fuck',
'Web Pix',
'Web Sauger',
+ 'Web spyder',
'Web Sucker',
+ 'web-capture\.net',
+ 'Web-sniffer',
'Webalta',
'Webauskunft',
'WebAuto',
@@ -1157,8 +1329,9 @@ class Crawlers extends AbstractProvider
'Webster',
'WebStripper',
'WebSucker',
- 'Webthumb\/',
+ 'webtech\/',
'WebThumbnail',
+ 'Webthumb\/',
'WebWhacker',
'WebZIP',
'WeLikeLinks',
@@ -1167,24 +1340,31 @@ class Crawlers extends AbstractProvider
'wf84',
'Wfuzz\/',
'wget',
+ 'WhatCMS',
'WhatsApp',
'WhatsMyIP',
'WhatWeb',
'WhereGoes\?',
'Whibse',
+ 'WhoAPI\/',
'WhoRunsCoinHive',
'Whynder Magnet',
'Windows-RSS-Platform',
+ 'WinHttp-Autoproxy-Service',
+ 'WinHTTP\/',
'WinPodder',
'wkhtmlto',
'wmtips',
'Woko',
+ 'Wolfram HTTPClient',
'woorankreview',
- 'Word\/',
'WordPress\/',
'WordupinfoSearch',
+ 'Word\/',
+ 'worldping-api',
'wotbox',
'WP Engine Install Performance API',
+ 'WP Rocket',
'wpif',
'wprecon\.com survey',
'WPScan',
@@ -1194,20 +1374,21 @@ class Crawlers extends AbstractProvider
'WWW-Mechanize',
'WWW::Document',
'WWW::Mechanize',
- 'www\.monitor\.us',
'WWWOFFLE',
+ 'www\.monitor\.us',
'x09Mozilla',
'x22Mozilla',
'XaxisSemanticsClassifier',
+ 'XenForo\/',
'Xenu Link Sleuth',
'XING-contenttabreceiver',
'xpymep([0-9]?)\.exe',
- 'Y!J-(ASR|BSC)',
- 'Y\!J-BRW',
+ 'Y!J-[A-Z][A-Z][A-Z]',
'Yaanb',
'yacy',
'Yahoo Link Preview',
'YahooCacheSystem',
+ 'YahooMailProxy',
'YahooYSMcm',
'YandeG',
'Yandex(?!Search)',
@@ -1215,6 +1396,7 @@ class Crawlers extends AbstractProvider
'yeti',
'Yo-yo',
'Yoleo Consumer',
+ 'yomins\.com',
'yoogliFetchAgent',
'YottaaMonitor',
'Your-Website-Sucks',
@@ -1226,15 +1408,17 @@ class Crawlers extends AbstractProvider
'Zao',
'Zauba',
'Zemanta Aggregator',
- 'Zend_Http_Client',
'Zend\\\\Http\\\\Client',
+ 'Zend_Http_Client',
'Zermelo',
'Zeus ',
'zgrab',
'ZnajdzFoto',
+ 'ZnHTTP',
'Zombie\.js',
'Zoom\.Mac',
+ 'ZoteroTranslationServer',
'ZyBorg',
- '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)',
+ '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)',
);
}
diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php
index e6b3ca89..62745572 100644
--- a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php
+++ b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php
@@ -67,6 +67,8 @@ class Exclusions extends AbstractProvider
'; IDbot',
'; ID bot',
'; POWER BOT',
- ';', // Remove the following characters ;
+ 'OCTOPUS-CORE',
+ 'htc_botdugls',
+ 'super\/\d+\/Android\/\d+',
);
}