From 52df5c69e47bc1cb43f8beb9a8cca275f1527bf5 Mon Sep 17 00:00:00 2001
From: Albert S <mail@quitesimple.org>
Date: Tue, 27 Aug 2019 13:12:40 +0200
Subject: [PATCH] fetchers: update browser version automatically. updated
 headers to look more like recent versions

---
 fetchers/chrome          | 9 ++++++---
 fetchers/chrome_agents   | 2 +-
 fetchers/chrome_version  | 1 +
 fetchers/firefox         | 9 ++++++---
 fetchers/firefox_agents  | 4 ++--
 fetchers/firefox_version | 1 +
 6 files changed, 17 insertions(+), 9 deletions(-)
 create mode 100644 fetchers/chrome_version
 create mode 100644 fetchers/firefox_version

diff --git a/fetchers/chrome b/fetchers/chrome
index 21fcfbe..5b7c3a3 100755
--- a/fetchers/chrome
+++ b/fetchers/chrome
@@ -4,7 +4,10 @@ if [ $# -ne 2 ] ; then
 echo "usage: $0 url output" 1>&2
 exit 1
 fi
-#better randomize
-useragent=$(shuf -n 1 $RANDRSS_ROOT/fetchers/chrome_agents)
+if find "$RANDRSS_ROOT/fetchers/chrome_version" -mtime +1 | grep -q chrome_version ; then
+curl "https://en.wikipedia.org/w/index.php?title=Template:Latest_stable_software_release/Google_Chrome&action=edit" | grep version1 | sed -e 's/.*version1 = \([0-9][0-9]\).\([0-9]\)\+.\([0-9]\+\)\.\([0-9]\+\)..*/\1.\2.\3.\4/g' > /"$RANDRSS_ROOT/fetchers/chrome_version"
+fi
+VERSION=$(cat "$RANDRSS_ROOT/fetchers/chrome_version")
+useragent=$(shuf -n 1 $RANDRSS_ROOT/fetchers/chrome_agents  | sed -e "s/VERSION/$VERSION/g")
 
-curl "$1" -H 'Accept-Encoding: gzip, deflate, br' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H "User-Agent: $useragent" -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'  -H 'Connection: keep-alive'  -H 'Cache-Control: max-age=0' --compressed > $2
+curl "$1" -H 'Connection: keep-alive' -H 'Upgrade-Insecure-Requests: 1' -H "$useragent" -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-User: ?1' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3' -H 'Sec-Fetch-Site: same-origin' -H 'Accept-Encoding: gzip, deflate, br' -H 'Accept-Language: en-US,en;q=0.9' --compressed > $2
diff --git a/fetchers/chrome_agents b/fetchers/chrome_agents
index e6f62bc..645f722 100644
--- a/fetchers/chrome_agents
+++ b/fetchers/chrome_agents
@@ -1 +1 @@
-Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/VERSION Safari/537.36 
diff --git a/fetchers/chrome_version b/fetchers/chrome_version
new file mode 100644
index 0000000..56c5c56
--- /dev/null
+++ b/fetchers/chrome_version
@@ -0,0 +1 @@
+76.0.3809.132
diff --git a/fetchers/firefox b/fetchers/firefox
index 11e9bc8..8a420f2 100755
--- a/fetchers/firefox
+++ b/fetchers/firefox
@@ -5,7 +5,10 @@ if [ $# -ne 2 ] ; then
 echo "usage: $0 url output" 1>&2
 exit 1
 fi
-#better randomize
-useragent=$(shuf -n 1 $RANDRSS_ROOT/fetchers/firefox_agents)
+if find "$RANDRSS_ROOT/fetchers/firefox_version" -mtime +1 | grep -q firefox_version ; then
+curl https://ftp.mozilla.org/pub/firefox/releases/ | grep releases | sed -e 's;.*releases/\(.*\)/".*;\1;g' | grep -v [a-z] | sort -V | tail -n 1 |  sed -e 's/\([0-9][0-9]\)\.\([0-9]\).*/\1.\2/g' > /"$RANDRSS_ROOT/fetchers/firefox_version"
+fi
+VERSION=$(cat "$RANDRSS_ROOT/fetchers/firefox_version")
+useragent=$(shuf -n 1 $RANDRSS_ROOT/fetchers/firefox_agents | sed -e "s/VERSION/$VERSION/g" )
 
-curl "$1" -H "User-Agent: $useragent" -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' --compressed -H 'Connection: keep-alive' -H 'Upgrade-Insecure-Requests: 1' > $2
+curl "$1" -H "User-Agent: $useragent" -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" -H "Accept-Language: en-US,en;q=0.5" --compressed -H "Connection: keep-alive" -H "Upgrade-Insecure-Requests: 1" -H "Cache-Control: max-age=0" > $2
diff --git a/fetchers/firefox_agents b/fetchers/firefox_agents
index 8af2681..41aa9bc 100644
--- a/fetchers/firefox_agents
+++ b/fetchers/firefox_agents
@@ -1,2 +1,2 @@
-Mozilla/5.0 (X11; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0
-Mozilla/5.0 (Windows NT 10.0; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0
+Mozilla/5.0 (X11; Linux x86_64; rv:VERSION) Gecko/20100101 Firefox/VERSION
+Mozilla/5.0 (Windows NT 10.0; WOW64; rv:VERSION) Gecko/20100101 Firefox/VERSION
diff --git a/fetchers/firefox_version b/fetchers/firefox_version
new file mode 100644
index 0000000..5eb735d
--- /dev/null
+++ b/fetchers/firefox_version
@@ -0,0 +1 @@
+68.0