Browse Source

Initial version

master
EonaCat 4 years ago
parent
commit
5f2187dc32
  1. BIN
      installer/blocky.jar
  2. 7
      installer/blocky.white
  3. 169
      installer/blockyConvert.php
  4. 39
      installer/conf/blocky
  5. 275
      installer/downloader.sh
  6. 65
      installer/install.sh
  7. 15
      installer/updateLists.sh
  8. 58
      nginx_example/blocky.eonacat.com

BIN
installer/blocky.jar

Binary file not shown.

7
installer/blocky.white

@ -0,0 +1,7 @@
# Blocky
# Blocking domains the way you want it.
# (c) EonaCat (Jeroen Saey) 2017 blocky
# https://blocky.eonacat.com
// WhiteList
// Example: http://www.google.com

169
installer/blockyConvert.php

@ -0,0 +1,169 @@
<?php
/* Blocky
* Blocking domains the way you want it.
* (c) EonaCat (Jeroen Saey) 2017 blocky
* https://blocky.eonacat.com
*/
// fetch data
$rawdata = explode("\n", trim(file_get_contents('/etc/blocky/domains/blocky.list')));
// replace double dots to one dot only
$rawdata = str_replace('..', '.',$rawdata);
echo count($rawdata), " Entries processed..\n";
// fetch whitelist
$whitelist = explode("\n", trim(file_get_contents('/etc/blocky/blockywhitelist')));
echo count($whitelist), " Whitelist Hosts processed..\n";
// utility function
function isWhitelisted($host){
global $whitelist;
foreach ($whitelist as $entry){
// wildcard ?
if (strpos($entry, '.') == 0){
// strip dot
$entry = substr($entry, 1);
if (substr($host, -strlen($entry)) === $entry){
return true;
}
// singlular match
}else{
// match found ?
if ($entry == $host){
return true;
}
}
}
// not in the list
return false;
}
// domains to hostname => [subdomain] array
$hosts = array();
$count = 0;
// Make sure all the values are still unqiue (should be)
$rawdata = array_unique($rawdata);
function checkurl($url) {
// First check: is the url just a domain name? (allow a slash at the end)
$_domain_regex = "|^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*(\.[A-Za-z]{2,})/?$|";
if (preg_match($_domain_regex, $url)) {
return true;
}
// Second: Check if it's a url with a scheme and all
$_regex = '#^([a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))$#';
if (preg_match($_regex, $url, $matches)) {
// pull out the domain name, and make sure that the domain is valid.
$_parts = parse_url($url);
if (!in_array($_parts['scheme'], array( 'http', 'https' )))
return false;
// Check the domain using the regex, stops domains like "-example.com" passing through
if (!preg_match($_domain_regex, $_parts['host']))
return false;
// This domain looks pretty valid. Only way to check it now is to download it!
return true;
}
return false;
}
// process data
foreach ($rawdata as $row)
{
// Set all the domain information to lowerCasing first
// (So we dont have duplicate domain names in different cases)
$row = "0.0.0.0 " . strtolower($row);
// valid line ? format <IP> <hostname>
$row = explode(" ", $row);
if (count($row) === 2 && checkurl($row[1]))
{
// host entry in whitelist ?
if (!isWhitelisted($row[1])){
// domain in list ?
if (!isset($hosts[$row[1]])){
$hosts[$row[1]] = array();
}
// push host
$hosts[$row[1]][] = $row[1];
}
} else
{
echo "Invalid domain => $row[1]\n";
}
}
echo "===============================\n";
echo "Total valid domains " . count(array_keys($hosts)) . "\n";
echo "Total invalid domains $count\n";
echo "Checked total domains " . count($rawdata) . "\n";
echo "================================\n\n";
echo "Formatting to DNSMasq\n";
// reduce array to dnsmasq format
$dnsmasq = '';
array_walk($hosts, function($hostnames, $domain) use (&$dnsmasq){
// generate local zone entry
foreach ($hostnames as $host){
$dnsmasq .= 'address=/' . $host . '/0.0.0.0'."\n";
}
});
echo "Formatting to unbound\n";
// reduce array to unbound format
$unbound = '';
array_walk($hosts, function($hostnames, $domain) use (&$unbound){
// generate local zone entry
foreach ($hostnames as $host)
{
$z = 'local-zone: "' . $host . '"' . ' redirect'."\n";
$z .= 'local-data: "' . $host . ' A 0.0.0.0"'."\n";
}
$unbound .= $z;
});
echo "Formatting to IPTables\n";
$iptables = '';
array_walk($hosts, function($hostnames, $domain) use (&$iptables){
// generate local zone entry
foreach ($hostnames as $host){
$iptables .= $host . "\n";
}
});
$ZONENAME = "blocky.null.zone";
$ZONEFILEPATH = "/etc/bind/$ZONENAME";
echo "Formatting to BIND (preconfigured with zoneFile location: $ZONENAME)\n";
$bind = '';
array_walk($hosts, function($hostnames, $domain) use (&$bind, $ZONEFILEPATH)
{
// generate local zone entry
foreach ($hostnames as $host)
{
$bind .= "zone \"" . $host . "\" { type master; notify no; file \"" . $ZONEFILEPATH . "\"; };" . "\n";
}
});
// save
file_put_contents('dnsmasq.blocky.conf', $dnsmasq);
file_put_contents('unbound.blocky.conf', $unbound);
file_put_contents('iptables.blocky.conf', $iptables);
file_put_contents('bind.blocky.conf', $bind);

39
installer/conf/blocky

@ -0,0 +1,39 @@
#! /bin/sh
# /etc/init.d/blocky
#
# Blocky
# Blocking domains the way you want it.
# (c) EonaCat (Jeroen Saey) 2017 blocky
# https://blocky.eonacat.com
startBlocky(){
echo "Starting blocky"
cd /etc/blocky
screen -d -m java -jar /etc/blocky/blocky.jar
echo "Blocky started"
}
stopBlocky(){
echo "Stopping blocky"
pkill -f blocky.jar
echo "Blocky stopped"
}
case "$1" in
start)
startBlocky
;;
stop)
stopBlocky
;;
restart)
stopBlocky
startBlocky
;;
*)
echo "Usage: /etc/init.d/blocky {start|restart|stop}"
exit 1
;;
esac
exit 0

275
installer/downloader.sh

@ -0,0 +1,275 @@
#!/usr/bin/env bash
# Blocky
# Blocking domains the way you want it.
# (c) EonaCat (Jeroen Saey) 2017 blocky
# https://blocky.eonacat.com
# Download the ad lists from internet
fn_exists() {
# appended double quote is an ugly trick to make sure we do get a string -- if $1 is not a known command, type does not output anything
[ `type -t $1`"" == 'function' ]
}
if ! fn_exists echoAndWriteToLog; then
function echoAndWriteToLog()
{
echo -e $1
}
fi
#Check for internet connectivity
test=google.com
if nc -zw1 $test 443 && echo |openssl s_client -connect $test:443 2>&1 |awk '
handshake && $1 == "Verification" { if ($2=="OK") exit; exit 1 }
$1 $2 == "SSLhandshake" { handshake = 1 }'
then
: #We have internet connection
else
echoAndWriteToLog "We dont have a proper internet connection, the blocky downloader cannot continue"
exit 1
fi
# Variables for various stages of downloading and formatting the list
basename=blocky
extension=domains
blockyDir=/etc/blocky
domainList=${blockyDir}/blockysetuplist
blockyDomainDownloadDir=${blockyDir}"/domains"
blockyTemp1=${basename}.blocky1.temp
blockyTemp2=${basename}.blocky2.temp
blockyTemp3=${basename}.blocky3.temp
blockingList=${blockyDir}/blocky.list
mkdir ${blockyDomainDownloadDir}
skipDownload=false
formatDomains()
{
echoAndWriteToLog "Formatting Blocky domains"
sources=()
while IFS= read -r line || [[ -n "$line" ]]; do
#Do not read commented out or blank lines
if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then
echoAndWriteToLog "" > /dev/null
else
sources+=(${line})
fi
done < ${domainList}
echoAndWriteToLog "OK"
}
# patternCheck - check to see if curl downloaded any new files.
checkDomainPatterns()
{
patternBuffer=$1
success=$2
error=$3
if [ $success = true ]; then
# check if download was successful but list has not been modified
if [ "${error}" == "304" ]; then
echoAndWriteToLog "No changes detected, download skipped!"
# check if the patternbuffer is a non-zero length file
elif [[ -s "${patternBuffer}" ]]; then
# Some of the blocklists are copyright, they need to be downloaded
# and stored as is. They can be processed for content after they
# have been saved.
mv "${patternBuffer}" "${saveLocation}"
echoAndWriteToLog "List updated, download successful!"
else
# Empty file -> use previously downloaded list
echoAndWriteToLog "Received empty file, using cached one (list not updated!)"
fi
else
# check if cached list exists
if [[ -r "${saveLocation}" ]]; then
echoAndWriteToLog "List download failed, using cached list (list not updated!)"
else
echoAndWriteToLog "Download failed and no cached list available (list will not be considered)"
fi
fi
}
# download - curl the specified url with any needed command extensions
downloadDomains()
{
url=$1
agent=$3
# create a temp file for the list
patternBuffer=$(mktemp)
downloadNewerOnly=""
if [[ -r ${saveLocation} ]]; then
# if domain has been saved, add file for date check to only download newer
downloadNewerOnly="-z ${saveLocation}"
fi
# Silently curl url
error=$(curl -s -L ${downloadNewerOnly} -w %{http_code} -A "${agent}" ${url} -o ${patternBuffer})
echoAndWriteToLog " done"
if [ ${error}="200" ] ; then
status="OK";
result=true;
elif [ ${error}="304" ] ; then
status="Not modified";
result=true;
elif [ ${error}="403" ] ; then
status="Forbidden";
result=false;
elif [ ${error}="404" ] ; then
status="Not found";
result=false;
elif [ ${error}="408" ] ; then
status="Time-out";
result=false;
elif [ ${error}="451" ] ; then
status="Unavailable For Legal Reasons";
result=false;
elif [ ${error}="500" ] ; then
status="Internal Server Error";
result=false;
elif [ ${error}="521" ] ; then
status="Unavailable For Legal Reasons";
result=false;
elif [ ${error}="522" ] ; then
status="Connection Timed Out (Cloudflare)";
result=false;
else
status="$error";
result=false;
fi
echoAndWriteToLog " Status: ${status}"
# Process result
checkDomainPatterns "${patternBuffer}" ${result} "${error}"
# Delete temp file if it hasn't been moved
if [[ -f "${patternBuffer}" ]]; then
rm "${patternBuffer}"
fi
}
# Prepare the domains for downloading
prepareDomainsForDownload()
{
agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2227.0 Safari/537.36 (Blocky)"
# download the setup list
saveLocation=${domainList}
downloadDomains "https://git.saey.me/EonaCat/BlockyRules/raw/branch/master/blockyrules" "$agent"
formatDomains
# Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and # blank lines
for ((i = 0; i < "${#sources[@]}"; i++)); do
url=${sources[$i]}
# Get just the domain from the URL
domain=$(echo "${url}" | cut -d'/' -f3)
# Save the file as blockyList.domain.#.tempBlockyListPart
saveLocation=${blockyDomainDownloadDir}/blockyList.${domain}.${i}.tempBlockyListPart
activeDomains[$i]=${saveLocation}
if [[ "${skipDownload}" == false ]]; then
echoAndWriteToLog "Retrieving list from $domain"
downloadDomains "$url" "$agent"
fi
done
}
# merge domains to one list
mergeDomainsToList()
{
echoAndWriteToLog ""
# Find all active domains and compile them into one file and remove CRs
echoAndWriteToLog "Merging lists of domains together"
truncate -s 0 ${blockyDomainDownloadDir}/${blockyTemp1}
for i in "${activeDomains[@]}"; do
if [[ -r "${i}" ]]; then
cat "${i}" | tr "[:upper:]" "[:lower:]" | tr -d '\r' >> ${blockyDomainDownloadDir}/${blockyTemp1}
fi
done
#Add populair advertisment ulrs (when we dont have any urls)
if [[ $(wc -l < "${blockyDomainDownloadDir}/${blockyTemp1}")] = "0" ]]; then
echoAndWriteToLog "Adding default hardcoded list of advertisments to blockage list because we did not receive any domains"
echo "doubleclick.net" >> ${blockyDomainDownloadDir}/${blockyTemp1}
echo "googlesyndication.com" >> ${blockyDomainDownloadDir}/${blockyTemp1}
echo "googleadservices.com" >> ${blockyDomainDownloadDir}/${blockyTemp1}
echo "google-analytics.com" >> ${blockyDomainDownloadDir}/${blockyTemp1}
echo "ads.youtube.com" >> ${blockyDomainDownloadDir}/${blockyTemp1}
echo "adserver.yahoo.com" >> ${blockyDomainDownloadDir}/${blockyTemp1}
echo "l.facebook.com" >> ${blockyDomainDownloadDir}/${blockyTemp1}
fi
echoAndWriteToLog "OK"
}
removeDuplicateDomains()
{
# Sort and remove duplicates
echoAndWriteToLog "Removing duplicate domains"
sort -u ${blockyDomainDownloadDir}/${blockyTemp2} > ${blockyDomainDownloadDir}/${blockyTemp3}
echoAndWriteToLog "OK"
unique=$(wc -l < ${blockyDomainDownloadDir}/${blockyTemp3})
echoAndWriteToLog "$unique unique domains"
}
doQuery ()
{
dig "$1" +noquestion +nostat +noanswer +noauthority 2> /dev/null
}
getAnswersNumber ()
{
local result=$(doQuery "$1")
result=${result##*ANSWER: }
echo "${result%%,*}"
}
makeDomainsValid()
{
echoAndWriteToLog "Removing all the comments and invalid domain names"
#Remove all lines that begin with special characters
#Remove all the comments (including inlines) lines as second
#Remove all the ipAddresses as thirth
#Remove all the tabs and spaces as fourth
#Replace multiple dots with 1 dot (e.g.: ..... > .) as fifth
#Remove dot if the line begins with a dot as last
cat ${blockyDomainDownloadDir}/${blockyTemp1} | \
sed -e 's/^[\<\>\(\)\{\}\;\$\%\,\*'\'']//' | \
sed 's:#.*$::g' | \
sed -e 's/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}//g' | \
sed -e 's/[\t ]//g;/^$/d' | \
sed 's/\.\.*/\./' | \
sed -e 's/^[.]//' > ${blockyDomainDownloadDir}/${blockyTemp2}
echoAndWriteToLog "OK"
numberOf=$(wc -l < ${blockyDomainDownloadDir}/${blockyTemp2})
echoAndWriteToLog "${numberOf} correct domains downloaded"
removeDuplicateDomains
}
moveBlockyList()
{
mv ${blockyDomainDownloadDir}/${blockyTemp3} ${blockyDomainDownloadDir}/blocky.list
}
#Download the domains
prepareDomainsForDownload
mergeDomainsToList
makeDomainsValid
moveBlockyList

65
installer/install.sh

@ -0,0 +1,65 @@
#!/bin/bash
# Blocky
# Blocking domains the way you want it.
# (c) EonaCat (Jeroen Saey) 2017 blocky
# https://blocky.eonacat.com
DIR=$(pwd)
BLOCKYDIR="/etc/blocky"
echo "Welcome to the Blocky installer"
echo "Blocky"
echo "Blocking domains the way you want it."
echo "(c) EonaCat (Jeroen Saey) 2017 blocky"
echo "https://blocky.eonacat.com"
echo "============================================"
if [ ! -d ${BLOCKYDIR} ]; then
mkdir -p $BLOCKYDIR
# echo "Installing NGINX"
# apt-get install nginx
echo "Installing PHP"
apt-get install php
echo "Installing Java"
sudo apt-get install default-jdk
apt-get install wget
echo "Setup blocky configuration"
cp $DIR/updateLists.sh $BLOCKYDIR
chmod +x $BLOCKYDIR/updateLists.sh
cp $DIR/blockyConvert.php $BLOCKYDIR
cp $DIR/downloader.sh $BLOCKYDIR
chmod +x $BLOCKYDIR/downloader.sh
cp $DIR/blocky.white $BLOCKYDIR
echo "Copying Blocky"
cp $DIR/blocky.jar $BLOCKYDIR
chmod +x $BLOCKYDIR/blocky.jar
cp $DIR/conf/blocky /etc/init.d/
chmod 755 /etc/init.d/blocky
echo "Adding to startup"
update-rc.d blocky defaults
echo '* 12 * * * '$BLOCKYDIR'/updateLists.sh' > /etc/cron.d/blocky
#wget https://blocky.lea.vc/installed.php > /dev/null 2>&1
echo "Blocky was installed successfully"
else
echo "Looks like the blocky prerequisites are already installed. Skipping installation of blocky prerequisites."
fi
bash $DIR/updateLists.sh
/etc/init.d/blocky stop
/etc/init.d/blocky start
echo "Blocky is running and updated"

15
installer/updateLists.sh

@ -0,0 +1,15 @@
#!/bin/bash
# Blocky
# Blocking domains the way you want it.
# (c) EonaCat (Jeroen Saey) 2017 blocky
# https://blocky.eonacat.com
DIR="/etc/blocky"
if [ ! -d "$DIR" ]; then
mkdir -p $DIR
fi
bash downloader.sh
php blockyConvert.php
cp iptables.blocky.conf /etc/blocky/blocky.black
/etc/init.d/blocky restart

58
nginx_example/blocky.eonacat.com

@ -0,0 +1,58 @@
upstream blocky {
server 127.0.0.1:80;
}
server {
server_name blocky.eonacat.com www.blocky.eonacat.com;
listen 7000;
#include the acme-challenge location
include snippets/acme.conf;
access_log /var/log/nginx/accessblockyeonacatcom.log;
error_log /var/log/nginx/errorblockyeonacatcom.log;
return 301 https://$server_name$request_uri;
}
server {
server_name blocky.eonacat.com www.blocky.eonacat.com;
access_log /var/log/nginx/accessblockyeonacatcom.log;
error_log /var/log/nginx/errorblockyeonacatcom.log;
# SSL configuration
listen 443 ssl;
#include the acme-challenge location
include snippets/acme.conf;
location / {
#try_files $uri $uri/ =404;
proxy_pass http://blocky;
proxy_set_header Accept-Encoding "";
sub_filter "http://blocky" "http://blocky";
proxy_pass_header Server;
sub_filter_once off;
}
location ~ /ad {
proxy_pass http://blocky;
proxy_set_header Accept-Encoding "";
proxy_pass_header Server;
sub_filter "http://blocky" "http://blocky";
sub_filter_once off;
}
#include PHP fastcgi config
include snippets/php.conf;
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
location ~ /\.ht {
deny all;
}
ssl_certificate /etc/letsencrypt/live/blocky.eonacat.com/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/blocky.eonacat.com/privkey.pem; # managed by Certbot
}
Loading…
Cancel
Save