initial Commit with Files from Blacktop

Dockerfile

@@ -0,0 +1,21 @@
+FROM gliderlabs/alpine:3.4
+
+MAINTAINER blacktop, https://github.com/blacktop
+
+# Add scripts
+COPY nsrl /nsrl
+RUN apk-install tini
+RUN apk-install -t .build-deps gcc libc-dev python-dev py-pip p7zip \
+  && set -x \
+  && apk --update add python $buildDeps \
+  && rm -f /var/cache/apk/* \
+  && pip install pybloom \
+  && /nsrl/shrink_nsrl.sh \
+  && apk del --purge .build-deps \
+  && rm -rf /tmp/* /root/.cache /var/cache/apk/* /nsrl/shrink_nsrl.sh
+
+WORKDIR /nsrl
+
+ENTRYPOINT ["/sbin/tini","--","/nsrl/search.py"]
+
+CMD ["-h"]

nsrl/build.py

@@ -0,0 +1,70 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+build.py
+~~~~~~~~
+
+This module builds a bloomfilter from the NSRL Whitelist Database.
+
+:copyright: (c) 2014 by Josh "blacktop" Maine.
+:license: MIT
+:improved_by: https://github.com/kost
+"""
+
+import binascii
+import os
+import sys
+
+from pybloom import BloomFilter
+
+nsrl_path = '/nsrl/NSRLFile.txt'
+error_rate = 0.01
+
+
+# reference - http://stackoverflow.com/a/9631635
+def blocks(this_file, size=65536):
+    while True:
+        b = this_file.read(size)
+        if not b:
+            break
+        yield b
+
+
+def main(argv):
+    if argv:
+        error_rate = float(argv[0])
+    print "[BUILDING] Using error-rate: {}".format(error_rate)
+    if os.path.isfile(nsrl_path):
+        print "[BUILDING] Reading in NSRL Database"
+        with open(nsrl_path) as f_line:
+            # Strip off header
+            _ = f_line.readline()
+            print "[BUILDING] Calculating number of hashes in NSRL..."
+            num_lines = sum(bl.count("\n") for bl in blocks(f_line))
+            print "[BUILDING] There are %s hashes in the NSRL Database" % num_lines
+        with open(nsrl_path) as f_nsrl:
+            # Strip off header
+            _ = f_nsrl.readline()
+            print "[BUILDING] Creating bloomfilter"
+            bf = BloomFilter(num_lines, error_rate)
+            print "[BUILDING] Inserting hashes into bloomfilter"
+            for line in f_nsrl:
+                md5_hash = line.split(",")[1].strip('"')
+                if md5_hash:
+                    try:
+                        md5 = binascii.unhexlify(md5_hash)
+                        bf.add(md5)
+                    except Exception as e:
+                        print "[ERROR] %s" % e
+            print "[BUILDING] NSRL bloomfilter contains {} items.".format(len(bf))
+            with open('nsrl.bloom', 'wb') as nb:
+                bf.tofile(nb)
+            print "[BUILDING] Complete"
+    else:
+        print("[ERROR] No such file or directory: %s", nsrl_path)
+
+    return
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

nsrl/search.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+search.py
+~~~~~~~~
+
+This module searches the bloomfilter for a given MD5 hash.
+
+:copyright: (c) 2014 by Josh "blacktop" Maine.
+:license: MIT
+:improved_by: https://github.com/kost
+"""
+
+import argparse
+import binascii
+
+from pybloom import BloomFilter
+
+
+def main():
+    parser = argparse.ArgumentParser(prog='blacktop/nsrl')
+    parser.add_argument("-v", "--verbose", help="Display verbose output message", action="store_true", required=False)
+    parser.add_argument('hash', metavar='MD5', type=str, nargs='+', help='a md5 hash to search for.')
+    args = parser.parse_args()
+
+    with open('nsrl.bloom', 'rb') as nb:
+        bf = BloomFilter.fromfile(nb)
+
+        for hash_hex in args.hash:
+            hash = binascii.unhexlify(hash_hex)
+            if args.verbose:
+                if hash in bf:
+                    print "Hash {} found in NSRL Database.".format(hash_hex)
+                else:
+                    print "Hash {} was NOT found in NSRL Database.".format(hash_hex)
+            else:
+                print hash in bf
+    return
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print "Error: %s" % e
+
+# test_hash = 'AABCA0896728846A9D5B841617EBE746'
+# calc_hash = '60B7C0FEAD45F2066E5B805A91F4F0FC'

nsrl/shrink_nsrl.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+# copyright: (c) 2014 by Josh "blacktop" Maine.
+# license: MIT
+
+set -x
+
+ERROR_RATE=0.01
+
+if [ -f /nsrl/*.zip ]; then
+   echo "File '.zip' Exists."
+else
+    echo "[INFO] Downloading NSRL Reduced Sets..."
+    NSRL_URL="http://www.nsrl.nist.gov/"
+    MIN_SET=$(wget -O - ${NSRL_URL}Downloads.htm 2> /dev/null | \
+      grep -m 1  "Minimal set" | \
+      grep -o '<a href=['"'"'"][^"'"'"']*['"'"'"]' | \
+      sed -e 's/^<a href=["'"'"']//' -e 's/["'"'"']$//')
+    wget -P /nsrl/ $NSRL_URL$MIN_SET 2> /dev/null
+fi
+
+echo "[INFO] Unzip NSRL Database zip to /nsrl/ ..."
+7za x -o/nsrl/ /nsrl/*.zip
+
+echo "[INFO] Build bloomfilter from NSRL Database ..."
+cd /nsrl && python /nsrl/build.py $ERROR_RATE
+echo "[INFO] Listing created files ..."
+ls -lah /nsrl
+
+echo "[INFO] Deleting all unused files ..."
+rm -f /nsrl/*.zip /nsrl/*.txt /nsrl/build.py
+ls -lah /nsrl