irt/docker_nsrl
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

changed shrink_nsrl to utilise new config file

Browse Source
This commit is contained in:
TKE
2020-04-15 16:35:01 +02:00
parent d7e7be0d88
commit 45f7d38019
1 changed files with 34 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
nsrl/shrink_nsrl.sh
View File

@@ -1,26 +1,44 @@
#!/bin/sh #!/bin/sh
# copyright: (c) 2014 by Josh "blacktop" Maine. rds_url=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=rds_url = )\S+')
# license: MIT version_url=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=version_url = )\S+')
ERROR_RATE=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=error_rate = )\S+')
baseurl="https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/current/" rds_name=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=rds_name = ).+')
rds_file="rds_modernm.zip" hashfile_name=$(cat /nsrl/nsrl.conf | grep -Pio '(?<=hashfile_name = ).+')
version_file="version.txt" build_date=$(date +%Y%M%d_%H%M)
set -x
ERROR_RATE=0.01
if [ -f /nsrl/*.zip ]; then if [ -f /nsrl/*.zip ]; then
echo "File '.zip' Exists." zip_filename=$(ls /nsrl/*.zip | head -n1)
echo "[INFO] ZIP-File Exists > ${zip_filename}"
zip_md5=$(md5sum "${zip_filename}" | cut -f1 -d" ")
rds_version=$(stat -c %y "${zip_filename}" )
else else
echo "[INFO] Downloading NSRL Reduced Sets..." zip_filename=${rds_url##*/}
wget -P /nsrl/ "${baseurl}${rds_file}" 2>/dev/null echo "[INFO] Downloading NSRL Sets (${rds_name}):"
wget -O /nsrl/version "${baseurl}${version_file}" 2>/dev/null echo "[INFO] URL: ${rds_url}"
#Downloading and Hashing at the same time
zip_md5=$(wget -O - "${rds_url}" 2>/dev/null | tee "/nsrl/${zip_filename}" | md5sum | cut -f1 -d" ")
wget -O /nsrl/.version "${version_url}" 2>/dev/null
rds_version=$(grep -F "${zip_md5}" /nsrl/.version | cut -f2 -d, | tr -d '"' )
fi fi
#update config
echo "build_date = ${build_date}" >> /nsrl/nsrl.conf
echo "zip_filename = ${zip_filename}" >> /nsrl/nsrl.conf
echo "zip_md5 = ${zip_md5}" >> /nsrl/nsrl.conf
echo "rds_version = ${rds_version}" >> /nsrl/nsrl.conf
echo "[INFO] Unzip NSRL Database zip to /nsrl/ ..." echo "[INFO] Unzip NSRL Database zip to /nsrl/ ..."
7za x -o/nsrl/ /nsrl/*.zip 7za x -o/nsrl/ "${zip_filename}"
echo "[INFO] Counting Hashes in /nsrl/${hashfile_name} ..."
hash_count=$(wc -l "/nsrl/${hashfile_name}")
#remove headline from count
(( hash_count=hash_count - 1 ))
echo "[INFO] /nsrl/${hashfile_name} contains ${hash_count} Hashes"
echo "hash_count = ${hash_count}" >> /nsrl/nsrl.conf
echo "[INFO] Build bloomfilter from NSRL Database ..." echo "[INFO] Build bloomfilter from NSRL Database ..."
cd /nsrl && python /nsrl/build.py $ERROR_RATE cd /nsrl && python /nsrl/build.py $ERROR_RATE
@@ -28,5 +46,5 @@ echo "[INFO] Listing created files ..."
ls -lah /nsrl ls -lah /nsrl
echo "[INFO] Deleting all unused files ..." echo "[INFO] Deleting all unused files ..."
rm -f /nsrl/*.zip /nsrl/*.txt /nsrl/build.py rm -f /nsrl/*.zip /nsrl/*.txt
ls -lah /nsrl ls -lah /nsrl