source: trunk/dnsbl/DNSBL.pm@ 10

Last change on this file since 10 was 10, checked in by Kris Deugau, 15 years ago

/trunk/dnsbl

Set IP count for listing a /31 to prevent errors on export

File size: 13.1 KB
Line 
1# DNSBL
2# Functions for interacting with the DNSBL database
3
4package DNSBL;
5
6use strict;
7use warnings;
8use Exporter;
9use DBI;
10use NetAddr::IP;
11
12use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
13
14$VERSION = 2.0;
15@ISA = qw(Exporter);
16@EXPORT_OK = qw(
17 );
18
19@EXPORT = (); # Export nothing by default.
20%EXPORT_TAGS = ( ALL => [qw(
21 )]
22 );
23
24## "constants"
25
26# 8 bits available
27# 128 is per-IP shitlist
28# 2 is IP hitlist
29# 1 not available so we don't $self->shoot(foot)
30our %bitfields = (
31 # block levels
32 0 => 16,
33 1 => 8,
34 2 => 4,
35 # ip
36 ip => 2,
37 # OOB
38 org => 32,
39 block => 64,
40 # "I'm a total spamming moron!" - per-IP only!
41 slist => 128
42);
43
44# probably needs some tuning; even 7 hits in a /24 is a pretty small percentage
45# number of IPs in a block of the given masklength needed to have that block automatically listed
46our %autolist = (
47 31 => 1,
48 30 => 1,
49 29 => 2,
50 28 => 3,
51 27 => 4,
52 26 => 5,
53 25 => 6,
54 24 => 7,
55 23 => 8,
56 22 => 10,
57 21 => 12,
58 20 => 14,
59 19 => 16,
60 18 => 18,
61 17 => 20,
62 16 => 22,
63 15 => 24,
64 14 => 26,
65 13 => 28,
66 12 => 30,
67 11 => 32,
68 10 => 34,
69 9 => 36,
70 8 => 38,
71 7 => 2**31,
72 6 => 2**31,
73 5 => 2**31,
74 4 => 2**31,
75 3 => 2**31,
76 2 => 2**31,
77 1 => 2**31,
78 0 => 2**31
79);
80
81# le sigh. constants for masklength iterationing
82our @howmany = (1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2);
83
84# variables
85our $dbh;
86
87our $err;
88our $errstr;
89
90# basic object subs
91sub new {
92# iff we want to start taking arguments, or doing other things on instantiation
93# my $self = {};
94# bless $self, "DNSBL";
95# return $self;
96 bless {};
97}
98
99sub DESTROY {
100 my $self = shift;
101 $self->dbclose();
102}
103
104# JIC someone wants to close the db but not finish the script
105sub dbclose {
106 $dbh->rollback;
107 $dbh->disconnect;
108}
109
110## specific object subs:
111
112sub connect {
113 my $DSN = "DBI:Pg:host=dbhost;dbname=dnsbl";
114# my $DSN = "DBI:Pg:dbname=dnsbl";
115 my $user = "dnsbl";
116 my $pass = "spambgone";
117 ## want to NOT autocommit everything, it's unlikely we'll step on our own toes but...
118 $dbh = DBI->connect($DSN, $user, $pass, {
119 AutoCommit => 0,
120 PrintError => 1
121 })
122 or die "database inaccessible: ".$DBI::errstr;
123 return $dbh;
124}
125
126
127## DNSBL::ipexists()
128# return report count if the IP has been reported, otherwise return undef
129sub ipexists {
130 my $self = shift;
131 my $ip = shift;
132 my $sth = $dbh->prepare("SELECT count FROM iplist WHERE ip=?");
133 $sth->execute($ip);
134 my ($ret) = $sth->fetchrow_array();
135 return $ret;
136} # end ipexists()
137
138
139# report an IP or URI to the db
140# increments a hit counter iff the reported IP or URI exists, otherwise it adds it
141sub report {
142 my $self = shift;
143 my $rep = shift;
144 my $sth;
145 my $rows = 0;
146 if ($rep =~ /^[\d.]+$/) {
147 # weesa gonna ASS-U-ME IP addresses are sanely formatted.
148 $sth = $dbh->prepare("SELECT count FROM iplist WHERE ip=?");
149 $sth->execute($rep) or die "eep? ".$dbh->errstr."\n";
150 $rows = $sth->rows;
151 if ($rows == 0) {
152 $sth = $dbh->prepare("INSERT INTO iplist (ip) VALUES (?)");
153 } elsif ($rows == 1) {
154 $sth = $dbh->prepare("UPDATE iplist SET count=count+1 WHERE ip=?");
155 } else {
156 die "db corrupt: found $rows matches on $rep\n";
157 }
158 $sth->execute($rep) or die "couldn't update listing for $rep: ".$dbh->errstr."\n";
159 } else {
160 return;
161 }
162 $dbh->commit;
163 return $rows;
164} # end report()
165
166
167# add a new org
168# return the orgid
169# if the org exists, return the orgid anyway
170sub addorg {
171 my $self = shift;
172 my $orgname = shift;
173 my $listme = shift || 'n';
174 my $ret = $self->orgexists($orgname);
175 return $ret if $ret;
176 my $sth = $dbh->prepare("INSERT INTO orgs (orgname,listme) VALUES (?,?)");
177 $sth->execute($orgname,$listme) or die "couldn't add org $orgname: ".$dbh->errstr."\n";
178 $dbh->commit;
179 $sth = $dbh->prepare("SELECT orgid FROM orgs WHERE orgname=?");
180 $sth->execute($orgname);
181 my ($orgid) = $sth->fetchrow_array();
182 return $orgid;
183} # end addorg
184
185
186# checks for existence - nb, exact match! No way to really handle anything else. :/
187sub orgexists {
188 my $self = shift;
189 my $org = shift;
190 my $sth = $dbh->prepare("SELECT orgid FROM orgs WHERE orgname=?");
191 $sth->execute($org);
192 my ($ret) = $sth->fetchrow_array();
193 return $ret;
194} # end orgexists();
195
196
197# add a block. requires the orgid
198##fixme needs error handling
199sub addblock {
200 my $self = shift;
201 my $blockin = shift;
202 my $orgid = shift;
203 my $level = shift;
204 $blockin =~ s/^\s+//;
205 $blockin =~ s/\s+$//;
206 my $block = new NetAddr::IP "$blockin"; # need this to clean up messes like ranges. sigh.
207
208 local $dbh->{AutoCommit} = 1; # force autocommit
209
210 my $sth = $dbh->prepare("INSERT INTO blocks (block,orgid,level) VALUES (?,?,?)");
211 $sth->execute("$block",$orgid,$level);
212 # nb: no need to return anything, since the CIDR block is the key
213}
214
215
216sub blockexists {
217 my $self = shift;
218 my $block = shift;
219 my $sth = $dbh->prepare("SELECT count(*) FROM blocks WHERE block=?");
220 $sth->execute($block);
221 my ($ret) = $sth->fetchrow_array();
222 return $ret;
223}
224
225
226# returns list (block,orgname) for the block that contains the passed IP.
227# accepts a level argument if you don't want the top-level registrar allocation block
228sub getcontainer {
229 my $self = shift;
230 my $ip = shift;
231 my $level = shift || 0;
232 my $sth = $dbh->prepare("SELECT b.block,o.orgname FROM blocks b INNER JOIN orgs o ".
233 "ON b.orgid=o.orgid WHERE b.block >> ? AND b.level = ?");
234 $sth->execute($ip,$level);
235 return $sth->fetchrow_array();
236} # end getcontainer()
237
238
239# whee! Recursion is Fun!
240# Call ourself to dig down through the layers of blocks from registar-allocation
241# (level 0) to final block (level n, not to exceed 2)
242# Take a reference to a hash, and stuff it full of blacklisting goodness.
243# Optionally accept a level, block-container, and OOB block and org arguments for
244# the container to check and return
245# Returns no value directly
246# Calls itself to walk down the tree of containers
247sub export {
248 my $self = shift;
249 my $listhosts = shift;
250
251# Export data as CIDR netblocks or classful (A/B/C) blocks
252# Assume classful as it's more compatible with different DNS servers
253 my $mode = shift || 'class';
254
255# Assume we're checking the whole enchilada if we don't get told where to look.
256 my $level = shift || 0;
257 my $container = shift || '0.0.0.0/0';
258 my $oobblock = shift || 0;
259 my $ooborg = shift || 0;
260
261 if ($level > 3) {
262 warn "getting too deep, breaking off! ($container, $level)\n";
263 return;
264 }
265
266# fiddle $container into a sane state.
267 if ($container =~ m|^\d+\.\d+\.\d+/\d+$|) {
268 $container =~ s|/(\d+)$|.0/$1|;
269 } elsif ($container =~ m|^\d+\.\d+/\d+$|) {
270 $container =~ s|/(\d+)$|.0.0/$1|;
271 } elsif ($container =~ m|^\d+/(\d+)$|) {
272 $container =~ s|/(\d+)$|.0.0.0/$1|;
273 }
274
275 my $sth = $dbh->prepare("SELECT count(*) FROM blocks WHERE block << ?");
276 $sth->execute($container);
277 my ($nblocks) = $sth->fetchrow_array();
278
279 if ($nblocks > 0) {
280 my $sql = "SELECT b.block,b.listme,o.orgname,o.listme ".
281 "FROM blocks b INNER JOIN orgs o ON b.orgid=o.orgid ".
282 "WHERE b.level=$level and b.block << '$container' ORDER BY b.block, masklen(b.block) DESC";
283 $sth = $dbh->prepare($sql);
284 $sth->execute();
285 while (my ($cidr,$listblock,$org,$listorg) = $sth->fetchrow_array()) {
286 $self->export($listhosts,$mode,$level+1,$cidr,$listblock,$listorg);
287 }
288 } # avoid checking content of subs if we don't have any
289
290 # don't check all 4.2 billion IPs individually if we're looking at all teh Intarwebs
291 return if $container eq '0.0.0.0/0';
292
293##fixme: need a way to dig out orphan IPs at all levels - IPs not found in a
294# subblock of the current container when the current container *has* subblocks
295# NB: this may be better handled as an out-of-band data-integrity-checker
296
297 # decrement level here so the right bitfield setting gets picked. this segment
298 # is inherently off-by-one from the block-recursion loop, and I can't see a
299 # better way to work around that. >:(
300 $level--;
301
302 # need this for a bunch of things, may as well do it here
303 my ($masklen) = ($container =~ m|/(\d+)$|);
304
305# Snag all parent block "is-it-listed?" data, and stuff it into a single
306# variable we can use later. Much faster than retrieving this data
307# individually, for each octet iteration.
308
309 my $mycount = 0;
310 my $sql = "SELECT count(i.ip),b.block,b.level,b.listme AS oobblock,o.listme AS ooborg ".
311 "FROM iplist i INNER JOIN blocks b ON i.ip << b.block INNER JOIN orgs o ON b.orgid = o.orgid ".
312 "WHERE b.block >>= ? ".
313 "GROUP BY b.block,b.level,b.listme,o.listme ORDER BY b.block";
314 my $parsth = $dbh->prepare($sql);
315 $parsth->execute($container);
316 my $pdata = 0;
317 while (my ($pcount,$p,$plev,$pblock,$porg) = $parsth->fetchrow_array) {
318 my ($pmasklen) = ($p =~ m|\d+/(\d+)$|);
319 $pdata |= $bitfields{$plev} if $pcount >= $autolist{$pmasklen};
320 $pdata |= $bitfields{block} if $pblock;
321 $pdata |= $bitfields{org} if $porg;
322 $mycount = $pcount if $p eq $container;
323 }
324
325 if ($mode eq 'cidr') {
326 $listhosts->{$container} |= $pdata if $pdata && ($ooborg || $oobblock || ($mycount >= $autolist{$masklen}));
327 } else {
328 # if $cidr->masklen is <= 24, iterate on /24 boundaries for bulk sublisting
329 # if $cidr->masklen is <= 16, iterate on /16 boundaries for bulk sublisting
330 # if $cidr->masklen is <= 8, iterate on /8 boundaries for bulk sublisting
331
332 if ($pdata) {
333 my @blocksubs;
334 if ($masklen <= 30 && $masklen > 24) {
335 my ($net,$octet) = ($container =~ m|^(\d+\.\d+\.\d+\.)(\d+)/|);
336 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
337 my $host = "$net$entry";
338 $listhosts->{$host} = 0 if !defined($listhosts->{$host});
339 $listhosts->{$host} |= $pdata;
340 }
341 } elsif ($masklen <= 24 && $masklen > 16) {
342 my ($net,$octet) = ($container =~ m|^(\d+\.\d+\.)(\d+)\.\d+/|);
343 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
344 my $twofour = "$net$entry.*";
345 $listhosts->{$twofour} |= $pdata;
346 }
347 } elsif ($masklen <= 16 && $masklen > 8) {
348 my ($net,$octet) = ($container =~ m|^(\d+\.)(\d+)\.\d+\.\d+/|);
349 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
350 my $sixteen = "$net$entry.*";
351 $listhosts->{$sixteen} |= $pdata;
352 }
353 } elsif ($masklen <= 8) {
354 my ($octet) = ($container =~ m|^(\d+)\.\d+\.\d+\.\d+/|);
355 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
356 my $eight = "$entry.*";
357 $listhosts->{$eight} |= $pdata;
358 }
359 }
360
361#print "DEBUG1: $container, ".(@blocksubs + 0)."\n";
362# this seems to be a BIG timesink... execution time ~1:30 without, ~4:30 with
363#if (0){
364# $sth = $dbh->prepare("select block,level,listme from blocks where block >> ?");
365# my $sth2 = $dbh->prepare("select count(*) from iplist where ip << ?");
366# foreach (@blocksubs) {
367#print " DEBUG: $_ container-is-listed check\n";
368# collect info on container block(s)
369# $sth->execute($container);
370# while (my ($parent, $plev, $listme) = $sth->fetchrow_array()) {
371# $sth2->execute($parent);
372# my ($parlen) = ($parent =~ m|/(\d+)|);
373# my ($parcount) = $sth2->fetchrow_array();
374#print " DEBUG: $parent: $parlen, $parcount, $plev\n";
375# $listhosts->{$_} |= $bitfields{$plev} if $parcount >= $autolist{$parlen}; #hmm.
376# $listhosts->{$_} |= $bitfields{block} if $listme;
377# }
378# }
379#}
380
381 } # generate autolist entries for ips/octets not (yet) seen in reports
382
383 } # cidr vs classful mode
384
385 $sth = $dbh->prepare("SELECT ip,s4list FROM iplist WHERE ip << ? ORDER BY ip");
386 $sth->execute($container);
387 while (my ($ip,$moron) = $sth->fetchrow_array()) {
388 $listhosts->{$ip} |= $pdata;
389 if ($moron) {
390 $listhosts->{$ip} = $bitfields{slist};
391 } else {
392 $listhosts->{$ip} |= $bitfields{ip};
393 }
394 }
395
396# get IPs which for reasons unknown are apparently allocated directly from the
397# parent registry (and so do not have containing netblocks in this system) O_o
398# select * from iplist where not (select count(*) from blocks where ip << block) > 0;
399
400 return;
401} # end export()
402
403
404sub export_alt {
405 my $self = shift;
406 my $listhosts = shift;
407 my $level = shift || 0;
408 my $container = shift || '0.0.0.0/0';
409 my $oobblock = shift || 0;
410 my $ooborg = shift || 0;
411
412#print "\nDEBUG: called with $level, $container, $oobblock, $ooborg\n";
413# if $level > 2 or $container =~ /^64\.76\./;
414# my %listhosts;
415
416# $level = 0 if !$level;
417 if ($level > 3) {
418 warn "getting too deep, breaking off!\n";
419 return;
420 }
421
422 my $sth = $dbh->prepare("select ip,s4list from iplist order by ip");
423 my $bsth = $dbh->prepare("select b.block,b.listme,b.level,o.listme ".
424 "from blocks b inner join orgs o on b.orgid=o.orgid ".
425 "where b.block >> ?");
426 while (my ($ip,$s4list) = $sth->fetchrow_array) {
427 $bsth->execute($ip);
428 while (my ($block,$blisted,$blevel,$olisted) = $bsth->fetchrow_array) {
429 $listhosts->{$ip} |= 0;
430 }
431 }
432
433} # end export_alt()
434
435
436## DNSBL::autolist_block()
437# check if a block should be autolisted
438sub autolist_block {
439 my $self = shift;
440 my $block = shift;
441
442 my $cidr = new NetAddr::IP "$block";
443 my $sth = $dbh->prepare("select count(*) from iplist where ip << ?");
444 $sth->execute("$cidr");
445 my ($count) = $sth->fetchrow_array;
446
447 return 1 if $count >= $autolist{$cidr->masklen};
448 return 0;
449} # end autolist_block()
450
451
452# make Perl happy
4531;
Note: See TracBrowser for help on using the repository browser.