source: trunk/dnsbl/DNSBL.pm@ 22

Last change on this file since 22 was 15, checked in by Kris Deugau, 14 years ago

/trunk/dnsbl

Bump thresholds for autolisting a block at /21 or larger

File size: 13.2 KB
Line 
1# DNSBL
2# Functions for interacting with the DNSBL database
3
4package DNSBL;
5
6use strict;
7use warnings;
8use Exporter;
9use DBI;
10use NetAddr::IP;
11
12use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
13
14$VERSION = 2.0;
15@ISA = qw(Exporter);
16@EXPORT_OK = qw(
17 );
18
19@EXPORT = (); # Export nothing by default.
20%EXPORT_TAGS = ( ALL => [qw(
21 )]
22 );
23
24## "constants"
25
26# 8 bits available
27# 128 is per-IP shitlist
28# 2 is IP hitlist
29# 1 not available so we don't $self->shoot(foot)
30our %bitfields = (
31 # block levels
32 0 => 16,
33 1 => 8,
34 2 => 4,
35 # ip
36 ip => 2,
37 # OOB
38 org => 32,
39 block => 64,
40 # "I'm a total spamming moron!" - per-IP only!
41 slist => 128
42);
43
44# probably needs some tuning; even 7 hits in a /24 is a pretty small percentage
45# number of IPs in a block of the given masklength needed to have that block automatically listed
46our %autolist = (
47 31 => 1,
48 30 => 1,
49 29 => 2,
50 28 => 3,
51 27 => 4,
52 26 => 5,
53 25 => 6,
54 24 => 7,
55 23 => 8,
56 22 => 10,
57 21 => 13,
58 20 => 16,
59 19 => 19,
60 18 => 22,
61 17 => 26,
62 16 => 30,
63 15 => 34,
64 14 => 38,
65 13 => 42,
66 12 => 46,
67 11 => 50,
68 10 => 54,
69 9 => 58,
70 8 => 62,
71 7 => 2**31,
72 6 => 2**31,
73 5 => 2**31,
74 4 => 2**31,
75 3 => 2**31,
76 2 => 2**31,
77 1 => 2**31,
78 0 => 2**31
79);
80
81# le sigh. constants for masklength iterationing
82our @howmany = (1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2);
83
84# variables
85our $dbh;
86
87our $err;
88our $errstr;
89
90# basic object subs
91sub new {
92# iff we want to start taking arguments, or doing other things on instantiation
93# my $self = {};
94# bless $self, "DNSBL";
95# return $self;
96 bless {};
97}
98
99sub DESTROY {
100 my $self = shift;
101 $self->dbclose();
102}
103
104# JIC someone wants to close the db but not finish the script
105sub dbclose {
106 $dbh->rollback;
107 $dbh->disconnect;
108}
109
110## specific object subs:
111
112sub connect {
113 my $DSN = "DBI:Pg:host=dbhost;dbname=dnsbl";
114# my $DSN = "DBI:Pg:dbname=dnsbl";
115 my $user = "dnsbl";
116 my $pass = "spambgone";
117 ## want to NOT autocommit everything, it's unlikely we'll step on our own toes but...
118 $dbh = DBI->connect($DSN, $user, $pass, {
119 AutoCommit => 0,
120 PrintError => 1
121 })
122 or die "database inaccessible: ".$DBI::errstr;
123 return $dbh;
124}
125
126
127## DNSBL::ipexists()
128# return report count if the IP has been reported, otherwise return undef
129sub ipexists {
130 my $self = shift;
131 my $ip = shift;
132 my $sth = $dbh->prepare("SELECT count FROM iplist WHERE ip=?");
133 $sth->execute($ip);
134 my ($ret) = $sth->fetchrow_array();
135 return $ret;
136} # end ipexists()
137
138
139# report an IP or URI to the db
140# increments a hit counter iff the reported IP or URI exists, otherwise it adds it
141sub report {
142 my $self = shift;
143 my $rep = shift;
144 my $sth;
145 my $rows = 0;
146 if ($rep =~ /^[\d.]+$/) {
147 # weesa gonna ASS-U-ME IP addresses are sanely formatted.
148 $sth = $dbh->prepare("SELECT count FROM iplist WHERE ip=?");
149 $sth->execute($rep) or die "eep? ".$dbh->errstr."\n";
150 $rows = $sth->rows;
151 if ($rows == 0) {
152 $sth = $dbh->prepare("INSERT INTO iplist (ip) VALUES (?)");
153 } elsif ($rows == 1) {
154 $sth = $dbh->prepare("UPDATE iplist SET count=count+1 WHERE ip=?");
155 } else {
156 die "db corrupt: found $rows matches on $rep\n";
157 }
158 $sth->execute($rep) or die "couldn't update listing for $rep: ".$dbh->errstr."\n";
159 } else {
160 return;
161 }
162 $dbh->commit;
163 return $rows;
164} # end report()
165
166
167# add a new org
168# return the orgid
169# if the org exists, return the orgid anyway
170sub addorg {
171 my $self = shift;
172 my $orgname = shift;
173 my $listme = shift || 'n';
174 my $ret = $self->orgexists($orgname);
175 return $ret if $ret;
176 my $sth = $dbh->prepare("INSERT INTO orgs (orgname,listme) VALUES (?,?)");
177 $sth->execute($orgname,$listme) or die "couldn't add org $orgname: ".$dbh->errstr."\n";
178 $dbh->commit;
179 $sth = $dbh->prepare("SELECT orgid FROM orgs WHERE orgname=?");
180 $sth->execute($orgname);
181 my ($orgid) = $sth->fetchrow_array();
182 return $orgid;
183} # end addorg
184
185
186# checks for existence - nb, exact match! No way to really handle anything else. :/
187sub orgexists {
188 my $self = shift;
189 my $org = shift;
190 my $sth = $dbh->prepare("SELECT orgid FROM orgs WHERE orgname=?");
191 $sth->execute($org);
192 my ($ret) = $sth->fetchrow_array();
193 return $ret;
194} # end orgexists();
195
196
197# add a block. requires the orgid
198##fixme needs error handling
199sub addblock {
200 my $self = shift;
201 my $blockin = shift;
202 my $orgid = shift;
203 my $level = shift;
204 $blockin =~ s/^\s+//;
205 $blockin =~ s/\s+$//;
206 my $block = new NetAddr::IP "$blockin"; # need this to clean up messes like ranges. sigh.
207
208 return "$blockin not a single CIDR range" if !$block;
209
210 local $dbh->{AutoCommit} = 1; # force autocommit
211
212 my $sth = $dbh->prepare("INSERT INTO blocks (block,orgid,level) VALUES (?,?,?)");
213 $sth->execute("$block",$orgid,$level);
214 return $sth->errstr if $sth->err;
215 # nb: no need to return anything, since the CIDR block is the key
216}
217
218
219sub blockexists {
220 my $self = shift;
221 my $block = shift;
222 my $sth = $dbh->prepare("SELECT count(*) FROM blocks WHERE block=?");
223 $sth->execute($block);
224 my ($ret) = $sth->fetchrow_array();
225 return $ret;
226}
227
228
229# returns list (block,orgname) for the block that contains the passed IP.
230# accepts a level argument if you don't want the top-level registrar allocation block
231sub getcontainer {
232 my $self = shift;
233 my $ip = shift;
234 my $level = shift || 0;
235 my $sth = $dbh->prepare("SELECT b.block,o.orgname FROM blocks b INNER JOIN orgs o ".
236 "ON b.orgid=o.orgid WHERE b.block >> ? AND b.level = ?");
237 $sth->execute($ip,$level);
238 return $sth->fetchrow_array();
239} # end getcontainer()
240
241
242# whee! Recursion is Fun!
243# Call ourself to dig down through the layers of blocks from registar-allocation
244# (level 0) to final block (level n, not to exceed 2)
245# Take a reference to a hash, and stuff it full of blacklisting goodness.
246# Optionally accept a level, block-container, and OOB block and org arguments for
247# the container to check and return
248# Returns no value directly
249# Calls itself to walk down the tree of containers
250sub export {
251 my $self = shift;
252 my $listhosts = shift;
253
254# Export data as CIDR netblocks or classful (A/B/C) blocks
255# Assume classful as it's more compatible with different DNS servers
256 my $mode = shift || 'class';
257
258# Assume we're checking the whole enchilada if we don't get told where to look.
259 my $level = shift || 0;
260 my $container = shift || '0.0.0.0/0';
261 my $oobblock = shift || 0;
262 my $ooborg = shift || 0;
263
264 if ($level > 3) {
265 warn "getting too deep, breaking off! ($container, $level)\n";
266 return;
267 }
268
269# fiddle $container into a sane state.
270 if ($container =~ m|^\d+\.\d+\.\d+/\d+$|) {
271 $container =~ s|/(\d+)$|.0/$1|;
272 } elsif ($container =~ m|^\d+\.\d+/\d+$|) {
273 $container =~ s|/(\d+)$|.0.0/$1|;
274 } elsif ($container =~ m|^\d+/(\d+)$|) {
275 $container =~ s|/(\d+)$|.0.0.0/$1|;
276 }
277
278 my $sth = $dbh->prepare("SELECT count(*) FROM blocks WHERE block << ?");
279 $sth->execute($container);
280 my ($nblocks) = $sth->fetchrow_array();
281
282 if ($nblocks > 0) {
283 my $sql = "SELECT b.block,b.listme,o.orgname,o.listme ".
284 "FROM blocks b INNER JOIN orgs o ON b.orgid=o.orgid ".
285 "WHERE b.level=$level and b.block << '$container' ORDER BY b.block, masklen(b.block) DESC";
286 $sth = $dbh->prepare($sql);
287 $sth->execute();
288 while (my ($cidr,$listblock,$org,$listorg) = $sth->fetchrow_array()) {
289 $self->export($listhosts,$mode,$level+1,$cidr,$listblock,$listorg);
290 }
291 } # avoid checking content of subs if we don't have any
292
293 # don't check all 4.2 billion IPs individually if we're looking at all teh Intarwebs
294 return if $container eq '0.0.0.0/0';
295
296##fixme: need a way to dig out orphan IPs at all levels - IPs not found in a
297# subblock of the current container when the current container *has* subblocks
298# NB: this may be better handled as an out-of-band data-integrity-checker
299
300 # decrement level here so the right bitfield setting gets picked. this segment
301 # is inherently off-by-one from the block-recursion loop, and I can't see a
302 # better way to work around that. >:(
303 $level--;
304
305 # need this for a bunch of things, may as well do it here
306 my ($masklen) = ($container =~ m|/(\d+)$|);
307
308# Snag all parent block "is-it-listed?" data, and stuff it into a single
309# variable we can use later. Much faster than retrieving this data
310# individually, for each octet iteration.
311
312 my $mycount = 0;
313 my $sql = "SELECT count(i.ip),b.block,b.level,b.listme AS oobblock,o.listme AS ooborg ".
314 "FROM iplist i INNER JOIN blocks b ON i.ip << b.block INNER JOIN orgs o ON b.orgid = o.orgid ".
315 "WHERE b.block >>= ? ".
316 "GROUP BY b.block,b.level,b.listme,o.listme ORDER BY b.block";
317 my $parsth = $dbh->prepare($sql);
318 $parsth->execute($container);
319 my $pdata = 0;
320 while (my ($pcount,$p,$plev,$pblock,$porg) = $parsth->fetchrow_array) {
321 my ($pmasklen) = ($p =~ m|\d+/(\d+)$|);
322 $pdata |= $bitfields{$plev} if $pcount >= $autolist{$pmasklen};
323 $pdata |= $bitfields{block} if $pblock;
324 $pdata |= $bitfields{org} if $porg;
325 $mycount = $pcount if $p eq $container;
326 }
327
328 if ($mode eq 'cidr') {
329 $listhosts->{$container} |= $pdata if $pdata && ($ooborg || $oobblock || ($mycount >= $autolist{$masklen}));
330 } else {
331 # if $cidr->masklen is <= 24, iterate on /24 boundaries for bulk sublisting
332 # if $cidr->masklen is <= 16, iterate on /16 boundaries for bulk sublisting
333 # if $cidr->masklen is <= 8, iterate on /8 boundaries for bulk sublisting
334
335 if ($pdata) {
336 my @blocksubs;
337 if ($masklen <= 30 && $masklen > 24) {
338 my ($net,$octet) = ($container =~ m|^(\d+\.\d+\.\d+\.)(\d+)/|);
339 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
340 my $host = "$net$entry";
341 $listhosts->{$host} = 0 if !defined($listhosts->{$host});
342 $listhosts->{$host} |= $pdata;
343 }
344 } elsif ($masklen <= 24 && $masklen > 16) {
345 my ($net,$octet) = ($container =~ m|^(\d+\.\d+\.)(\d+)\.\d+/|);
346 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
347 my $twofour = "$net$entry.*";
348 $listhosts->{$twofour} |= $pdata;
349 }
350 } elsif ($masklen <= 16 && $masklen > 8) {
351 my ($net,$octet) = ($container =~ m|^(\d+\.)(\d+)\.\d+\.\d+/|);
352 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
353 my $sixteen = "$net$entry.*";
354 $listhosts->{$sixteen} |= $pdata;
355 }
356 } elsif ($masklen <= 8) {
357 my ($octet) = ($container =~ m|^(\d+)\.\d+\.\d+\.\d+/|);
358 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
359 my $eight = "$entry.*";
360 $listhosts->{$eight} |= $pdata;
361 }
362 }
363
364#print "DEBUG1: $container, ".(@blocksubs + 0)."\n";
365# this seems to be a BIG timesink... execution time ~1:30 without, ~4:30 with
366#if (0){
367# $sth = $dbh->prepare("select block,level,listme from blocks where block >> ?");
368# my $sth2 = $dbh->prepare("select count(*) from iplist where ip << ?");
369# foreach (@blocksubs) {
370#print " DEBUG: $_ container-is-listed check\n";
371# collect info on container block(s)
372# $sth->execute($container);
373# while (my ($parent, $plev, $listme) = $sth->fetchrow_array()) {
374# $sth2->execute($parent);
375# my ($parlen) = ($parent =~ m|/(\d+)|);
376# my ($parcount) = $sth2->fetchrow_array();
377#print " DEBUG: $parent: $parlen, $parcount, $plev\n";
378# $listhosts->{$_} |= $bitfields{$plev} if $parcount >= $autolist{$parlen}; #hmm.
379# $listhosts->{$_} |= $bitfields{block} if $listme;
380# }
381# }
382#}
383
384 } # generate autolist entries for ips/octets not (yet) seen in reports
385
386 } # cidr vs classful mode
387
388 $sth = $dbh->prepare("SELECT ip,s4list FROM iplist WHERE ip << ? ORDER BY ip");
389 $sth->execute($container);
390 while (my ($ip,$moron) = $sth->fetchrow_array()) {
391 $listhosts->{$ip} |= $pdata;
392 if ($moron) {
393 $listhosts->{$ip} = $bitfields{slist};
394 } else {
395 $listhosts->{$ip} |= $bitfields{ip};
396 }
397 }
398
399# get IPs which for reasons unknown are apparently allocated directly from the
400# parent registry (and so do not have containing netblocks in this system) O_o
401# select * from iplist where not (select count(*) from blocks where ip << block) > 0;
402
403 return;
404} # end export()
405
406
407sub export_alt {
408 my $self = shift;
409 my $listhosts = shift;
410 my $level = shift || 0;
411 my $container = shift || '0.0.0.0/0';
412 my $oobblock = shift || 0;
413 my $ooborg = shift || 0;
414
415#print "\nDEBUG: called with $level, $container, $oobblock, $ooborg\n";
416# if $level > 2 or $container =~ /^64\.76\./;
417# my %listhosts;
418
419# $level = 0 if !$level;
420 if ($level > 3) {
421 warn "getting too deep, breaking off!\n";
422 return;
423 }
424
425 my $sth = $dbh->prepare("select ip,s4list from iplist order by ip");
426 my $bsth = $dbh->prepare("select b.block,b.listme,b.level,o.listme ".
427 "from blocks b inner join orgs o on b.orgid=o.orgid ".
428 "where b.block >> ?");
429 while (my ($ip,$s4list) = $sth->fetchrow_array) {
430 $bsth->execute($ip);
431 while (my ($block,$blisted,$blevel,$olisted) = $bsth->fetchrow_array) {
432 $listhosts->{$ip} |= 0;
433 }
434 }
435
436} # end export_alt()
437
438
439## DNSBL::autolist_block()
440# check if a block should be autolisted
441sub autolist_block {
442 my $self = shift;
443 my $block = shift;
444
445 my $cidr = new NetAddr::IP "$block";
446 my $sth = $dbh->prepare("select count(*) from iplist where ip << ?");
447 $sth->execute("$cidr");
448 my ($count) = $sth->fetchrow_array;
449
450 return 1 if $count >= $autolist{$cidr->masklen};
451 return 0;
452} # end autolist_block()
453
454
455# make Perl happy
4561;
Note: See TracBrowser for help on using the repository browser.