From 45f7d38019678116fc4560ec192c734eb3acd3b8 Mon Sep 17 00:00:00 2001
From: TKE <tobias.kessels@certbw.de>
Date: Wed, 15 Apr 2020 16:35:01 +0200
Subject: [PATCH] changed shrink_nsrl to utilise new config file

---
 nsrl/shrink_nsrl.sh | 50 ++++++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/nsrl/shrink_nsrl.sh b/nsrl/shrink_nsrl.sh
index a81fd96..a084f17 100755
--- a/nsrl/shrink_nsrl.sh
+++ b/nsrl/shrink_nsrl.sh
@@ -1,26 +1,44 @@
 #!/bin/sh
 
-# copyright: (c) 2014 by Josh "blacktop" Maine.
-# license: MIT
-
-baseurl="https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/current/"
-rds_file="rds_modernm.zip"
-version_file="version.txt"
-
-set -x
-
-ERROR_RATE=0.01
+rds_url=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=rds_url = )\S+')
+version_url=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=version_url = )\S+')
+ERROR_RATE=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=error_rate = )\S+')
+rds_name=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=rds_name = ).+')
+hashfile_name=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=hashfile_name = ).+')
+build_date=$(date +%Y%M%d_%H%M)
 
 if [ -f /nsrl/*.zip ]; then
-   echo "File '.zip' Exists."
+  zip_filename=$(ls /nsrl/*.zip | head -n1)
+  echo "[INFO] ZIP-File Exists > ${zip_filename}"
+  zip_md5=$(md5sum "${zip_filename}" | cut -f1 -d" ")
+  rds_version=$(stat -c %y "${zip_filename}" )
 else
-    echo "[INFO] Downloading NSRL Reduced Sets..."
-    wget -P /nsrl/ "${baseurl}${rds_file}" 2>/dev/null
-    wget -O /nsrl/version "${baseurl}${version_file}" 2>/dev/null
+  zip_filename=${rds_url##*/}
+  echo "[INFO] Downloading NSRL Sets (${rds_name}):"
+  echo "[INFO] URL: ${rds_url}"
+  #Downloading and Hashing at the same time
+  zip_md5=$(wget -O - "${rds_url}" 2>/dev/null | tee "/nsrl/${zip_filename}" | md5sum | cut -f1 -d" ")
+  wget -O /nsrl/.version "${version_url}" 2>/dev/null
+  rds_version=$(grep -F "${zip_md5}" /nsrl/.version | cut -f2 -d, | tr -d '"' )
 fi
 
+#update config
+echo "build_date = ${build_date}" >> /nsrl/nsrl.conf
+echo "zip_filename = ${zip_filename}" >> /nsrl/nsrl.conf
+echo "zip_md5 = ${zip_md5}" >> /nsrl/nsrl.conf
+echo "rds_version = ${rds_version}" >> /nsrl/nsrl.conf
+
+
 echo "[INFO] Unzip NSRL Database zip to /nsrl/ ..."
-7za x -o/nsrl/ /nsrl/*.zip
+7za x -o/nsrl/ "${zip_filename}"
+
+echo "[INFO] Counting Hashes in /nsrl/${hashfile_name} ..."
+hash_count=$(wc -l "/nsrl/${hashfile_name}")
+#remove headline from count
+(( hash_count=hash_count - 1 ))
+echo "[INFO] /nsrl/${hashfile_name} contains ${hash_count} Hashes"
+echo "hash_count = ${hash_count}" >> /nsrl/nsrl.conf
+
 
 echo "[INFO] Build bloomfilter from NSRL Database ..."
 cd /nsrl && python /nsrl/build.py $ERROR_RATE
@@ -28,5 +46,5 @@ echo "[INFO] Listing created files ..."
 ls -lah /nsrl
 
 echo "[INFO] Deleting all unused files ..."
-rm -f /nsrl/*.zip /nsrl/*.txt /nsrl/build.py
+rm -f /nsrl/*.zip /nsrl/*.txt
 ls -lah /nsrl