From e7e0c6200fc7e6bcb9fff6778440b77b29dfe573 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 23 Nov 2016 09:37:49 -0500 Subject: [PATCH 1/8] add initial base code for baddie detector --- baddie-detector/.gitignore | 10 + baddie-detector/README.md | 17 + baddie-detector/netdb/__init__.py | 1 + baddie-detector/netdb/__main__.py | 12 + baddie-detector/netdb/netdb.py | 366 ++++++++++++++++++++ baddie-detector/requirements.txt | 2 + baddie-detector/setup.py | 13 + baddie-detector/test.py | 2 + baddie-detector/tests/README | 10 + baddie-detector/tests/fuzzdb/TO_APPEASE_GIT | 0 baddie-detector/tests/test_netdb.py | 66 ++++ 11 files changed, 499 insertions(+) create mode 100644 baddie-detector/.gitignore create mode 100644 baddie-detector/README.md create mode 100644 baddie-detector/netdb/__init__.py create mode 100644 baddie-detector/netdb/__main__.py create mode 100644 baddie-detector/netdb/netdb.py create mode 100644 baddie-detector/requirements.txt create mode 100644 baddie-detector/setup.py create mode 100644 baddie-detector/test.py create mode 100644 baddie-detector/tests/README create mode 100644 baddie-detector/tests/fuzzdb/TO_APPEASE_GIT create mode 100644 baddie-detector/tests/test_netdb.py diff --git a/baddie-detector/.gitignore b/baddie-detector/.gitignore new file mode 100644 index 0000000..e1edc85 --- /dev/null +++ b/baddie-detector/.gitignore @@ -0,0 +1,10 @@ +*~ +*.pyc +*\#* + +v/ + +*.egg-info/ + +build/ +dist/ \ No newline at end of file diff --git a/baddie-detector/README.md b/baddie-detector/README.md new file mode 100644 index 0000000..5abe938 --- /dev/null +++ b/baddie-detector/README.md @@ -0,0 +1,17 @@ +# baddie detector + +i2p netdb blacklist generator tool + +## usage + + +building: + + python3 -m venv v + v/bin/pip install -r requirements.txt + v/bin/python setup.py install + + +running: + + v/bin/python -m baddiefinder diff --git a/baddie-detector/netdb/__init__.py b/baddie-detector/netdb/__init__.py new file mode 100644 index 0000000..b16d9c8 --- /dev/null +++ b/baddie-detector/netdb/__init__.py @@ -0,0 +1 @@ +from .netdb import inspect diff --git a/baddie-detector/netdb/__main__.py b/baddie-detector/netdb/__main__.py new file mode 100644 index 0000000..d3adbfd --- /dev/null +++ b/baddie-detector/netdb/__main__.py @@ -0,0 +1,12 @@ +# +# main driver +# + + +from .netdb import inspect + +def print_entry(ent): + print (ent) + +if __name__ == '__main__': + inspect(hook=print_entry) diff --git a/baddie-detector/netdb/netdb.py b/baddie-detector/netdb/netdb.py new file mode 100644 index 0000000..db597aa --- /dev/null +++ b/baddie-detector/netdb/netdb.py @@ -0,0 +1,366 @@ +## +## i2p netdb parser +## +## Author: Jeff +## MIT Liecense 2014 +## +import os,sys,struct,time,hashlib,fnmatch,io +from geoip import geolite2 +import base64 +import logging + +b64encode = lambda x : base64.b64encode(x, b'~-').decode('ascii') + +def sha256(data,raw=True): + """ + compute sha256 of data + """ + h = hashlib.new('sha256') + h.update(data) + if raw: + return h.digest() + else: + return h.hexdigest() + +class Inspector: + + _log = logging.getLogger('NDBInspector') + + def inspect(self, entry): + pass + + def run(self, ndb): + entry_counter = 0 + for root, dirnames, filenames in os.walk(ndb): + for filename in fnmatch.filter(filenames, '*.dat'): + fname = os.path.join(root, filename) + e = Entry(fname) + e.verify() + if e.valid: + entry_counter += 1 + self.inspect(e) + else: + self._log.warn('invalid entry in file {}'.format(fname)) + self._log.info('read {} entries'.format(entry_counter)) + +class Address: + """ + netdb address + """ + cost = None + transport = None + options = None + expire = None + location = None + + def valid(self): + return None not in (self.cost, self.transport, self.options, self.expire) + + def __repr__(self): + return 'Address: transport={} cost={} expire={} options={} location={} firewalled={}' \ + .format(self.transport, self.cost, self.expire, self.options, self.location, self.firewalled) + +class Entry: + """ + netdb entry + """ + _pubkey_size = 256 + _signkey_size = 128 + _min_cert_size = 3 + + _log = logging.getLogger('NDBEntry') + + @staticmethod + def _read_short(fd): + Entry._log.debug('read_short') + d = Entry._read(fd, 2) + if d: + return struct.unpack('!H',d)[0] + + @staticmethod + def _read_mapping(fd): + Entry._log.debug('read_mapping') + mapping = dict() + tsize = Entry._read_short(fd) + if tsize is None: + return + data = Entry._read(fd, tsize) + if data is None: + return + sfd = io.BytesIO(data) + ind = 0 + while ind < tsize: + Entry._log.debug(ind) + key = Entry._read_string(sfd) + if key is None: + return + Entry._log.debug(['key', key]) + + ind += len(key) + 2 + Entry._read_byte(sfd) + val = Entry._read_string(sfd) + if val is None: + return + Entry._log.debug(['val',val]) + + ind += len(val) + 2 + Entry._read_byte(sfd) + + #key = key[:-1] + #val = val[:-1] + if key in mapping: + v = mapping[key] + if isinstance(v,list): + mapping[key].append(val) + else: + mapping[key] = [v,val] + else: + mapping[key] = val + return mapping + + @staticmethod + def _read(fd, amount): + dat = fd.read(amount) + Entry._log.debug('read %d of %d bytes' % (len(dat), amount)) + if len(dat) == amount: + return dat + + + @staticmethod + def _read_byte(fd): + b = Entry._read(fd,1) + if b: + return struct.unpack('!B', b)[0] + + @staticmethod + def _read_two_bytes(fd): + b = Entry._read(fd,2) + if b: + return struct.unpack('!H', b)[0] + + @staticmethod + def _read_string(fd): + Entry._log.debug('read_string') + slen = Entry._read_byte(fd) + if slen: + return Entry._read(fd, slen) + + @staticmethod + def _read_time(fd): + d = Entry._read(fd, 8) + if d: + li = struct.unpack('!Q', d)[0] + return li + + @staticmethod + def _read_addr(fd): + """ + load next router address + """ + Entry._log.debug('read_addr') + addr = Address() + addr.cost = Entry._read_byte(fd) + addr.expire = Entry._read_time(fd) + addr.transport = Entry._read_string(fd) + addr.options = Entry._read_mapping(fd) + addr.firewalled = False + if addr.valid(): + # This is a try because sometimes hostnames show up. + # TODO: Make it allow host names. + try: + addr.location = geolite2.lookup(addr.options.get('host', None)) + except: + addr.location = None + + # If the router is firewalled (i.e. has no 'host' mapping), then use the first introducer (of 3). + # In the future it might be worth it to do something else, but this helps for geopip information for now. + # http://i2p-projekt.i2p/en/docs/transport/ssu#ra + if not addr.location: + # If there are introducers then it's probably firewalled. + addr.firewalled = True + try: + addr.location = geolite2.lookup(addr.options.get('ihost0', None)) + except: + addr.location = None + + return addr + + def __init__(self, filename): + """ + construct a NetDB Entry from a file + """ + self.addrs = list() + self.options = dict() + self.pubkey = None + self.signkey = None + self.cert = None + self.published = None + self.signature = None + self.peer_size = None + self.valid = False + try: + with open(filename, 'rb') as fr: + self._log.debug('load from file {}'.format(filename)) + self._load(fr) + #self.routerHash = + except (IOError, OSError) as e: + self._log.debug('load from file {} failed'.format(filename)) + + def _load(self, fd): + """ + load from file descriptor + More docs: http://i2p-projekt.i2p/en/docs/spec/common-structures#struct_RouterInfo + """ + + # router identity http://i2p-projekt.i2p/en/docs/spec/common-structures#struct_RouterIdentity + # Do not assume that these are always 387 bytes! + # There are 387 bytes plus the certificate length specified at bytes 385-386, which may be non-zero. + + # Subtract because read the Certificate on it's own. + data = self._read(fd, 387-self._min_cert_size) + if data is None: + return + ind = 0 + + # public key + self.pubkey = sha256(data[ind:ind+self._pubkey_size]) + ind += self._pubkey_size + + # signing key (we hash this later due to RI changes in 0.9.12) + self.signkey = data[ind:ind+self._signkey_size] + ind + self._signkey_size + + # certificate + self.cert = dict() + + # If it's not null, follow what happens here: http://i2p-projekt.i2p/en/docs/spec/common-structures#type_Certificate + cert_type = self._read_byte(fd) + cert_len = self._read_two_bytes(fd) + if cert_type == 5 and cert_len != 0: # New format where extra information is in the cert. + spkt = self._read_two_bytes(fd) + cpkt = self._read_two_bytes(fd) + if spkt == 0: + self.cert['signature_type'],cert_padding,cert_extra = 'DSA_SHA1',0,0 + elif spkt == 1: + self.cert['signature_type'],cert_padding,cert_extra = 'ECDSA_SHA256_P256',64,0 + elif spkt == 2: + self.cert['signature_type'],cert_padding,cert_extra = 'ECDSA_SHA384_P384',32,0 + elif spkt == 3: + self.cert['signature_type'],cert_padding,cert_extra = 'ECDSA_SHA512_P521',0,4 + elif spkt == 4: + self.cert['signature_type'],cert_padding,cert_extra = 'RSA_SHA256_2048',0,128 + elif spkt == 5: + self.cert['signature_type'],cert_padding,cert_extra = 'RSA_SHA384_3072',0,256 + elif spkt == 6: + self.cert['signature_type'],cert_padding,cert_extra = 'RSA_SHA512_4096',0,384 + elif spkt == 7: + self.cert['signature_type'],cert_padding,cert_extra = 'EdDSA_SHA512_Ed25519',96,0 + else: + Entry._log.debug('Bad cert sign type.') + return + + # This is always going to be 0 (as of 0.9.19), but future versions can add more crypto types. + if cpkt == 0: + self.cert['crypto_type'] = 'ElGamal' + else: + Entry._log.debug('Bad cert crypto type.') + return + else: # Old format where information is all in the main part. + self.cert['signature_type'],cert_padding,cert_extra = 'DSA_SHA1',0,0 + self.cert['crypto_type'] = 'ElGamal' + + # Parse public key properly (http://i2p-projekt.i2p/en/docs/spec/common-structures#type_Certificate) + if cert_padding > 0: + self.signkey = self.signkey[cert_padding:] + if cert_extra > 0: + self.signkey += self._read(fd,cert_extra) + + Entry._log.debug('parsed cert, sig type {}, crypto type {}.'.format(self.cert['signature_type'], self.cert['crypto_type'])) + + self.signkey = sha256(self.signkey) + + # date published + self.published = self._read_time(fd) + if self.published is None: + return + + # reachable addresses + self.addrs = list() + addrlen = self._read_byte(fd) + if addrlen is None: + return + for n in range(addrlen): + addr = self._read_addr(fd) + if addr is None: + return + self.addrs.append(addr) + + # peer size + self.peer_size = self._read_byte(fd) + if self.peer_size is None: + return + + # other options + self.options = self._read_mapping(fd) + if self.options is None: + return + + # signature + self.signature = sha256(self._read(fd, 40)) + if self.signature is None: + return + self.valid = True + + def verify(self): + """ + verify router identity + """ + #TODO: verify + + def __repr__(self): + val = str() + val += 'NetDB Entry ' + val += 'pubkey={} '.format(b64encode(self.pubkey)) + val += 'signkey={} '.format(b64encode(self.signkey)) + val += 'options={} '.format(self.options) + val += 'addrs={} '.format(self.addrs) + val += 'cert={} '.format(self.cert) + val += 'published={} '.format(self.published) + val += 'signature={}'.format(b64encode(self.signature)) + return val + + def dict(self): + """ + return dictionary in old format + """ + return dict({ + 'pubkey':b64encode(self.pubkey), + 'signkey':b64encode(self.signkey), + 'options':self.options, + 'addrs':self.addrs, + 'cert':self.cert, + 'published':self.published, + 'signature':b64encode(self.signature) + }) + +def inspect(hook=None,netdb_dir=os.path.join(os.environ['HOME'],'.i2pd','netDb')): + """ + iterate through the netdb + + parameters: + + hook - function taking 1 parameter + - the 1 parameter is a dictionary containing the info + of a netdb enrty + - called on every netdb entry + + netdb_dir - path to netdb folder + - defaults to $HOME/.i2pd/netDb/ + + """ + + insp = Inspector() + if hook is not None: + insp.inspect = hook + insp.run(netdb_dir) + diff --git a/baddie-detector/requirements.txt b/baddie-detector/requirements.txt new file mode 100644 index 0000000..8273fd3 --- /dev/null +++ b/baddie-detector/requirements.txt @@ -0,0 +1,2 @@ +python-geoip +python-geoip-geolite2 diff --git a/baddie-detector/setup.py b/baddie-detector/setup.py new file mode 100644 index 0000000..0d56706 --- /dev/null +++ b/baddie-detector/setup.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +from setuptools import setup + +setup(name = 'baddiedetector', + version = '0.0', + description = 'i2p netdb blocklist ', + author = 'Jeff Becker', + author_email = 'ampernand@gmail.com', + install_requires = ['python-geoip','python-geoip-geolite2'], + tests_require=['pytest'], + url = 'https://github.com/purplei2p/i2pd-tools', + packages = ['netdb', 'baddiefinder'], +) diff --git a/baddie-detector/test.py b/baddie-detector/test.py new file mode 100644 index 0000000..53b2a7b --- /dev/null +++ b/baddie-detector/test.py @@ -0,0 +1,2 @@ +import netdb +netdb.inspect() diff --git a/baddie-detector/tests/README b/baddie-detector/tests/README new file mode 100644 index 0000000..d4afa2a --- /dev/null +++ b/baddie-detector/tests/README @@ -0,0 +1,10 @@ +This requires pytest to be installed. +It's not a dependency in setup.py because end users should not have to worry about testing. + +Here is how to install it: + + http://pytest.org/latest/getting-started.html#installation + +And to run it: + + py.test diff --git a/baddie-detector/tests/fuzzdb/TO_APPEASE_GIT b/baddie-detector/tests/fuzzdb/TO_APPEASE_GIT new file mode 100644 index 0000000..e69de29 diff --git a/baddie-detector/tests/test_netdb.py b/baddie-detector/tests/test_netdb.py new file mode 100644 index 0000000..5360d62 --- /dev/null +++ b/baddie-detector/tests/test_netdb.py @@ -0,0 +1,66 @@ +# test_netdb.py - Test netdb.py +# Author: Chris Barry +# License: MIT + +# Note: this uses py.test. + +import netdb,os,random + +''' +def test_inspect(): + netdb.inspect() +''' + +def test_sha256(): + assert('d2f4e10adac32aeb600c2f57ba2bac1019a5c76baa65042714ed2678844320d0' == netdb.netdb.sha256('i2p is cool', raw=False)) + +def test_address_valid(): + invalid = netdb.netdb.Address() + valid = netdb.netdb.Address() + valid.cost = 10 + valid.transport = 'SSU' + valid.options = {'host': '0.0.0.0', 'port': '1234', 'key': '', 'caps': ''} + valid.expire = 0 + assert(valid.valid() and not invalid.valid()) + +def test_address_repr(): + valid = netdb.netdb.Address() + valid.cost = 10 + valid.transport = 'SSU' + valid.options = {'host': '0.0.0.0', 'port': '1234', 'key': '', 'caps': ''} + valid.expire = 0 + assert(repr(valid) == 'Address: transport=SSU cost=10 expire=0 options={\'host\': \'0.0.0.0\', \'port\': \'1234\', \'key\': \'\', \'caps\': \'\'} location=None') + +# TODO: test_entry* + +def test_entry_read_short(): + assert(True) +def test_entry_read_mapping(): + assert(True) +def test_entry_read(): + assert(True) +def test_entry_read_short(): + assert(True) +def test_entry_read_byte(): + assert(True) +def test_entry_read_string(): + assert(True) +def test_entry_init(): + assert(True) +def test_entry_load(): + assert(True) +def test_entry_verify(): + assert(True) +def test_entry_repr(): + assert(True) +def test_entry_dict(): + assert(True) + +# Make some garbage files and hope they break things. +def test_fuzz(): + pwd = os.environ['PWD'] + for i in range(1,100): + with open('{}/fuzzdb/{}.dat'.format(pwd, i), 'wb') as fout: + fout.write(os.urandom(random.randint(2,400))) # replace 1024 with size_kb if not unreasonably large + # Now let's inspect the garbage. + netdb.inspect(netdb_dir='{}/fuzzdb/'.format(pwd)) From fe758eb1aefb1b123268054c48d85a78a8008c45 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 23 Nov 2016 10:23:27 -0500 Subject: [PATCH 2/8] more --- baddie-detector/baddiefinder/__main__.py | 29 ++++++++++++++++++++ baddie-detector/baddiefinder/filter.py | 32 +++++++++++++++++++++++ baddie-detector/baddiefinder/processor.py | 26 ++++++++++++++++++ baddie-detector/baddiefinder/settings.py | 11 ++++++++ baddie-detector/baddiefinder/util.py | 15 +++++++++++ baddie-detector/baddies.ini | 2 ++ baddie-detector/netdb/netdb.py | 12 ++++++--- baddie-detector/setup.py | 4 +-- 8 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 baddie-detector/baddiefinder/__main__.py create mode 100644 baddie-detector/baddiefinder/filter.py create mode 100644 baddie-detector/baddiefinder/processor.py create mode 100644 baddie-detector/baddiefinder/settings.py create mode 100644 baddie-detector/baddiefinder/util.py create mode 100644 baddie-detector/baddies.ini diff --git a/baddie-detector/baddiefinder/__main__.py b/baddie-detector/baddiefinder/__main__.py new file mode 100644 index 0000000..7a13cec --- /dev/null +++ b/baddie-detector/baddiefinder/__main__.py @@ -0,0 +1,29 @@ +# +# +# + +import netdb + + +from argparse import ArgumentParser as AP + +from . import settings +from . import filter +from . import processor + +def main(): + ap = AP() + ap.add_argument("--settings", default="baddies.ini") + + args = ap.parse_args() + s = settings.load(args.settings) + fmax = s.get("thresholds", "max_floodfills_per_ip", fallback=5) + f = filter.FloodfillFilter(fmax) + p = processor.BaddieProcessor([f]) + netdb.inspect(p.hook) + with open(s.get("output", "file", fallback="baddies.txt"), 'w') as f: + p.write_blocklist(f) + + +if __name__ == "__main__": + main() diff --git a/baddie-detector/baddiefinder/filter.py b/baddie-detector/baddiefinder/filter.py new file mode 100644 index 0000000..b23673e --- /dev/null +++ b/baddie-detector/baddiefinder/filter.py @@ -0,0 +1,32 @@ + +from . import util + +class Filter: + + name = "unnamed filter" + + def process(self, info): + """ + process an info and return True if it should be added to blocklist + any other return value will cause this info to NOT be added to blocklist + """ + +class FloodfillFilter(Filter): + + name = "floodfill sybil detector" + + def __init__(self, fmax): + self._floodfills = dict() + self.fmax = int(fmax) + + def process(self, info): + caps = util.getcaps(info) + if not caps: + return False + if b'f' not in caps: + return False + h = util.getaddress(info) + if h not in self._floodfills: + self._floodfills[h] = 0 + self._floodfills[h] += 1 + return self._floodfills[h] > self.fmax diff --git a/baddie-detector/baddiefinder/processor.py b/baddie-detector/baddiefinder/processor.py new file mode 100644 index 0000000..a3ca87c --- /dev/null +++ b/baddie-detector/baddiefinder/processor.py @@ -0,0 +1,26 @@ +from . import util + +import datetime + +class BaddieProcessor: + + def __init__(self, filters): + self._filters = filters + self._baddies = dict() + + + def hook(self, entry): + for f in self._filters: + if f.process(entry) is True: + self.add_baddie(entry, 'detected by {}'.format(f.name)) + + def add_baddie(self, entry, reason): + addr = util.getaddress(entry) + if addr not in self._baddies: + self._baddies[addr] = '' + self._baddies[addr] += reason + ' ' + + def write_blocklist(self, f): + f.write('# baddies blocklist generated on {}'.format(datetime.datetime.now())) + for k in self._baddies: + f.write('{}:{}\n'.format(self._baddies[k], k)) diff --git a/baddie-detector/baddiefinder/settings.py b/baddie-detector/baddiefinder/settings.py new file mode 100644 index 0000000..8b13afe --- /dev/null +++ b/baddie-detector/baddiefinder/settings.py @@ -0,0 +1,11 @@ +# +# baddiefinder settings wrapper +# + +from configparser import ConfigParser + +def load(fname): + c = ConfigParser() + with open(fname) as f: + c.read_file(f, fname) + return c diff --git a/baddie-detector/baddiefinder/util.py b/baddie-detector/baddiefinder/util.py new file mode 100644 index 0000000..64bf22f --- /dev/null +++ b/baddie-detector/baddiefinder/util.py @@ -0,0 +1,15 @@ +def getaddress(info): + """ + get ip address from router info dict + """ + for addr in info.addrs: + opts = addr.options + if b'host' in opts: + return opts[b'host'] + +def getcaps(info): + """ + extract router caps + """ + if b'caps' in info.options: + return info.options[b'caps'] diff --git a/baddie-detector/baddies.ini b/baddie-detector/baddies.ini new file mode 100644 index 0000000..5b2a330 --- /dev/null +++ b/baddie-detector/baddies.ini @@ -0,0 +1,2 @@ +[thresholds] +max_floodfills_per_ip = 2 \ No newline at end of file diff --git a/baddie-detector/netdb/netdb.py b/baddie-detector/netdb/netdb.py index db597aa..8de54f6 100644 --- a/baddie-detector/netdb/netdb.py +++ b/baddie-detector/netdb/netdb.py @@ -5,9 +5,11 @@ ## MIT Liecense 2014 ## import os,sys,struct,time,hashlib,fnmatch,io -from geoip import geolite2 import base64 import logging +import pygeoip + +geo = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPCity.dat') b64encode = lambda x : base64.b64encode(x, b'~-').decode('ascii') @@ -152,6 +154,10 @@ class Entry: li = struct.unpack('!Q', d)[0] return li + @staticmethod + def geolookup(entry): + return geo.record_by_addr(entry) + @staticmethod def _read_addr(fd): """ @@ -168,7 +174,7 @@ class Entry: # This is a try because sometimes hostnames show up. # TODO: Make it allow host names. try: - addr.location = geolite2.lookup(addr.options.get('host', None)) + addr.location = geolookup(addr.options.get('host', None)) except: addr.location = None @@ -179,7 +185,7 @@ class Entry: # If there are introducers then it's probably firewalled. addr.firewalled = True try: - addr.location = geolite2.lookup(addr.options.get('ihost0', None)) + addr.location = geolookup(addr.options.get('ihost0', None)) except: addr.location = None diff --git a/baddie-detector/setup.py b/baddie-detector/setup.py index 0d56706..70210d5 100644 --- a/baddie-detector/setup.py +++ b/baddie-detector/setup.py @@ -2,8 +2,8 @@ from setuptools import setup setup(name = 'baddiedetector', - version = '0.0', - description = 'i2p netdb blocklist ', + version = '0.1', + description = 'i2p netdb blocklist tool', author = 'Jeff Becker', author_email = 'ampernand@gmail.com', install_requires = ['python-geoip','python-geoip-geolite2'], From 3bbbf260942f01526d6ae089cd8cea55489a4e14 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 23 Nov 2016 10:23:47 -0500 Subject: [PATCH 3/8] fix --- baddie-detector/baddiefinder/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baddie-detector/baddiefinder/processor.py b/baddie-detector/baddiefinder/processor.py index a3ca87c..c9ffc94 100644 --- a/baddie-detector/baddiefinder/processor.py +++ b/baddie-detector/baddiefinder/processor.py @@ -21,6 +21,6 @@ class BaddieProcessor: self._baddies[addr] += reason + ' ' def write_blocklist(self, f): - f.write('# baddies blocklist generated on {}'.format(datetime.datetime.now())) + f.write('# baddies blocklist generated on {}\n'.format(datetime.datetime.now())) for k in self._baddies: f.write('{}:{}\n'.format(self._baddies[k], k)) From d0e76fa6ebfb01cd715ad106dbf26a8cded3096c Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 24 Nov 2016 08:22:55 -0500 Subject: [PATCH 4/8] update baddies detector --- baddie-detector/baddiefinder/processor.py | 9 ++++----- baddie-detector/baddiefinder/util.py | 4 +++- baddie-detector/baddies.ini | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/baddie-detector/baddiefinder/processor.py b/baddie-detector/baddiefinder/processor.py index c9ffc94..cec8951 100644 --- a/baddie-detector/baddiefinder/processor.py +++ b/baddie-detector/baddiefinder/processor.py @@ -10,15 +10,14 @@ class BaddieProcessor: def hook(self, entry): + now = datetime.datetime.now() for f in self._filters: if f.process(entry) is True: - self.add_baddie(entry, 'detected by {}'.format(f.name)) + self.add_baddie(entry, 'detected by {} on {}'.format(f.name, now.strftime("%c").replace(":",'-'))) def add_baddie(self, entry, reason): - addr = util.getaddress(entry) - if addr not in self._baddies: - self._baddies[addr] = '' - self._baddies[addr] += reason + ' ' + addr = util.getaddress(entry).decode('ascii') + self._baddies[addr] = reason def write_blocklist(self, f): f.write('# baddies blocklist generated on {}\n'.format(datetime.datetime.now())) diff --git a/baddie-detector/baddiefinder/util.py b/baddie-detector/baddiefinder/util.py index 64bf22f..3c4292d 100644 --- a/baddie-detector/baddiefinder/util.py +++ b/baddie-detector/baddiefinder/util.py @@ -5,7 +5,9 @@ def getaddress(info): for addr in info.addrs: opts = addr.options if b'host' in opts: - return opts[b'host'] + h = opts[b'host'] + if b':' not in h: + return h def getcaps(info): """ diff --git a/baddie-detector/baddies.ini b/baddie-detector/baddies.ini index 5b2a330..a558c3a 100644 --- a/baddie-detector/baddies.ini +++ b/baddie-detector/baddies.ini @@ -1,2 +1,2 @@ [thresholds] -max_floodfills_per_ip = 2 \ No newline at end of file +max_floodfills_per_ip = 3 \ No newline at end of file From fb77e988e57f13dabfd5c13b30ac02c61c024a11 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 24 Nov 2016 08:26:34 -0500 Subject: [PATCH 5/8] update logic --- baddie-detector/baddiefinder/filter.py | 9 +++++---- baddie-detector/baddiefinder/processor.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/baddie-detector/baddiefinder/filter.py b/baddie-detector/baddiefinder/filter.py index b23673e..09621a9 100644 --- a/baddie-detector/baddiefinder/filter.py +++ b/baddie-detector/baddiefinder/filter.py @@ -7,7 +7,7 @@ class Filter: def process(self, info): """ - process an info and return True if it should be added to blocklist + process an info and return a string representation of a reason to add to blocklist any other return value will cause this info to NOT be added to blocklist """ @@ -22,11 +22,12 @@ class FloodfillFilter(Filter): def process(self, info): caps = util.getcaps(info) if not caps: - return False + return if b'f' not in caps: - return False + return h = util.getaddress(info) if h not in self._floodfills: self._floodfills[h] = 0 self._floodfills[h] += 1 - return self._floodfills[h] > self.fmax + if self._floodfills[h] > self.fmax: + return '{} > {} floodfills per ip'.format(self._floodfills[h], self.fmax) diff --git a/baddie-detector/baddiefinder/processor.py b/baddie-detector/baddiefinder/processor.py index cec8951..634475a 100644 --- a/baddie-detector/baddiefinder/processor.py +++ b/baddie-detector/baddiefinder/processor.py @@ -12,8 +12,9 @@ class BaddieProcessor: def hook(self, entry): now = datetime.datetime.now() for f in self._filters: - if f.process(entry) is True: - self.add_baddie(entry, 'detected by {} on {}'.format(f.name, now.strftime("%c").replace(":",'-'))) + reason = f.process(entry) + if reason is not None: + self.add_baddie(entry, 'detected by {} on {} ({})'.format(f.name, now.strftime("%c").replace(":",'-'), reason)) def add_baddie(self, entry, reason): addr = util.getaddress(entry).decode('ascii') From e548512f1e84966a7f10643f76bf243e4f1c1347 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 24 Nov 2016 08:28:00 -0500 Subject: [PATCH 6/8] update geoip related stuff --- baddie-detector/netdb/netdb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/baddie-detector/netdb/netdb.py b/baddie-detector/netdb/netdb.py index 8de54f6..ac735dd 100644 --- a/baddie-detector/netdb/netdb.py +++ b/baddie-detector/netdb/netdb.py @@ -7,9 +7,12 @@ import os,sys,struct,time,hashlib,fnmatch,io import base64 import logging -import pygeoip -geo = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPCity.dat') +try: + import pygeoip + geo = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPCity.dat') +except: + geo = None b64encode = lambda x : base64.b64encode(x, b'~-').decode('ascii') @@ -156,7 +159,8 @@ class Entry: @staticmethod def geolookup(entry): - return geo.record_by_addr(entry) + if geo: + return geo.record_by_addr(entry) @staticmethod def _read_addr(fd): From 08779e3f89b9a5f2c1447cfe2323fd73bec9f99f Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 24 Nov 2016 08:30:45 -0500 Subject: [PATCH 7/8] update --- baddie-detector/baddiefinder/__main__.py | 4 ++-- baddie-detector/baddiefinder/processor.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/baddie-detector/baddiefinder/__main__.py b/baddie-detector/baddiefinder/__main__.py index 7a13cec..fa92c2f 100644 --- a/baddie-detector/baddiefinder/__main__.py +++ b/baddie-detector/baddiefinder/__main__.py @@ -22,8 +22,8 @@ def main(): p = processor.BaddieProcessor([f]) netdb.inspect(p.hook) with open(s.get("output", "file", fallback="baddies.txt"), 'w') as f: - p.write_blocklist(f) - + print ('wrote {} blocklist entries'.format(p.write_blocklist(f))) + if __name__ == "__main__": main() diff --git a/baddie-detector/baddiefinder/processor.py b/baddie-detector/baddiefinder/processor.py index 634475a..8eeefef 100644 --- a/baddie-detector/baddiefinder/processor.py +++ b/baddie-detector/baddiefinder/processor.py @@ -21,6 +21,9 @@ class BaddieProcessor: self._baddies[addr] = reason def write_blocklist(self, f): + wrote = 0 f.write('# baddies blocklist generated on {}\n'.format(datetime.datetime.now())) for k in self._baddies: f.write('{}:{}\n'.format(self._baddies[k], k)) + wrote += 1 + return wrote From e5c253bd5e3a8f49adaa7f940004566a1466f86d Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 24 Nov 2016 08:35:47 -0500 Subject: [PATCH 8/8] rename --- README.md | 9 +++++++++ {baddie-detector => baddiefinder}/.gitignore | 0 {baddie-detector => baddiefinder}/README.md | 0 .../baddiefinder/__main__.py | 0 {baddie-detector => baddiefinder}/baddiefinder/filter.py | 0 .../baddiefinder/processor.py | 0 .../baddiefinder/settings.py | 0 {baddie-detector => baddiefinder}/baddiefinder/util.py | 0 {baddie-detector => baddiefinder}/baddies.ini | 0 {baddie-detector => baddiefinder}/netdb/__init__.py | 0 {baddie-detector => baddiefinder}/netdb/__main__.py | 0 {baddie-detector => baddiefinder}/netdb/netdb.py | 0 {baddie-detector => baddiefinder}/requirements.txt | 0 {baddie-detector => baddiefinder}/setup.py | 0 {baddie-detector => baddiefinder}/test.py | 0 {baddie-detector => baddiefinder}/tests/README | 0 .../tests/fuzzdb/TO_APPEASE_GIT | 0 {baddie-detector => baddiefinder}/tests/test_netdb.py | 0 18 files changed, 9 insertions(+) rename {baddie-detector => baddiefinder}/.gitignore (100%) rename {baddie-detector => baddiefinder}/README.md (100%) rename {baddie-detector => baddiefinder}/baddiefinder/__main__.py (100%) rename {baddie-detector => baddiefinder}/baddiefinder/filter.py (100%) rename {baddie-detector => baddiefinder}/baddiefinder/processor.py (100%) rename {baddie-detector => baddiefinder}/baddiefinder/settings.py (100%) rename {baddie-detector => baddiefinder}/baddiefinder/util.py (100%) rename {baddie-detector => baddiefinder}/baddies.ini (100%) rename {baddie-detector => baddiefinder}/netdb/__init__.py (100%) rename {baddie-detector => baddiefinder}/netdb/__main__.py (100%) rename {baddie-detector => baddiefinder}/netdb/netdb.py (100%) rename {baddie-detector => baddiefinder}/requirements.txt (100%) rename {baddie-detector => baddiefinder}/setup.py (100%) rename {baddie-detector => baddiefinder}/test.py (100%) rename {baddie-detector => baddiefinder}/tests/README (100%) rename {baddie-detector => baddiefinder}/tests/fuzzdb/TO_APPEASE_GIT (100%) rename {baddie-detector => baddiefinder}/tests/test_netdb.py (100%) diff --git a/README.md b/README.md index 597fc6c..decab3a 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,15 @@ sudo apt-get install \ ## Tools included +### baddiefinder + +i2p netdb blocklist generator tool + +#### Usage + +see [here](baddiefinder) + + ### keygen Generate an i2p private key diff --git a/baddie-detector/.gitignore b/baddiefinder/.gitignore similarity index 100% rename from baddie-detector/.gitignore rename to baddiefinder/.gitignore diff --git a/baddie-detector/README.md b/baddiefinder/README.md similarity index 100% rename from baddie-detector/README.md rename to baddiefinder/README.md diff --git a/baddie-detector/baddiefinder/__main__.py b/baddiefinder/baddiefinder/__main__.py similarity index 100% rename from baddie-detector/baddiefinder/__main__.py rename to baddiefinder/baddiefinder/__main__.py diff --git a/baddie-detector/baddiefinder/filter.py b/baddiefinder/baddiefinder/filter.py similarity index 100% rename from baddie-detector/baddiefinder/filter.py rename to baddiefinder/baddiefinder/filter.py diff --git a/baddie-detector/baddiefinder/processor.py b/baddiefinder/baddiefinder/processor.py similarity index 100% rename from baddie-detector/baddiefinder/processor.py rename to baddiefinder/baddiefinder/processor.py diff --git a/baddie-detector/baddiefinder/settings.py b/baddiefinder/baddiefinder/settings.py similarity index 100% rename from baddie-detector/baddiefinder/settings.py rename to baddiefinder/baddiefinder/settings.py diff --git a/baddie-detector/baddiefinder/util.py b/baddiefinder/baddiefinder/util.py similarity index 100% rename from baddie-detector/baddiefinder/util.py rename to baddiefinder/baddiefinder/util.py diff --git a/baddie-detector/baddies.ini b/baddiefinder/baddies.ini similarity index 100% rename from baddie-detector/baddies.ini rename to baddiefinder/baddies.ini diff --git a/baddie-detector/netdb/__init__.py b/baddiefinder/netdb/__init__.py similarity index 100% rename from baddie-detector/netdb/__init__.py rename to baddiefinder/netdb/__init__.py diff --git a/baddie-detector/netdb/__main__.py b/baddiefinder/netdb/__main__.py similarity index 100% rename from baddie-detector/netdb/__main__.py rename to baddiefinder/netdb/__main__.py diff --git a/baddie-detector/netdb/netdb.py b/baddiefinder/netdb/netdb.py similarity index 100% rename from baddie-detector/netdb/netdb.py rename to baddiefinder/netdb/netdb.py diff --git a/baddie-detector/requirements.txt b/baddiefinder/requirements.txt similarity index 100% rename from baddie-detector/requirements.txt rename to baddiefinder/requirements.txt diff --git a/baddie-detector/setup.py b/baddiefinder/setup.py similarity index 100% rename from baddie-detector/setup.py rename to baddiefinder/setup.py diff --git a/baddie-detector/test.py b/baddiefinder/test.py similarity index 100% rename from baddie-detector/test.py rename to baddiefinder/test.py diff --git a/baddie-detector/tests/README b/baddiefinder/tests/README similarity index 100% rename from baddie-detector/tests/README rename to baddiefinder/tests/README diff --git a/baddie-detector/tests/fuzzdb/TO_APPEASE_GIT b/baddiefinder/tests/fuzzdb/TO_APPEASE_GIT similarity index 100% rename from baddie-detector/tests/fuzzdb/TO_APPEASE_GIT rename to baddiefinder/tests/fuzzdb/TO_APPEASE_GIT diff --git a/baddie-detector/tests/test_netdb.py b/baddiefinder/tests/test_netdb.py similarity index 100% rename from baddie-detector/tests/test_netdb.py rename to baddiefinder/tests/test_netdb.py