source: trunk/dnsbl/DNSBL.pm@ 24

Last change on this file since 24 was 24, checked in by Kris Deugau, 14 years ago

/trunk/dnsbl

Add code to override autolist thresholds from database table

File size: 12.8 KB
Line 
1# DNSBL
2# Functions for interacting with the DNSBL database
3
4package DNSBL;
5
6use strict;
7use warnings;
8use Exporter;
9use DBI;
10use NetAddr::IP;
11
12use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
13
14$VERSION = 2.0;
15@ISA = qw(Exporter);
16@EXPORT_OK = qw(
17 );
18
19@EXPORT = (); # Export nothing by default.
20%EXPORT_TAGS = ( ALL => [qw(
21 )]
22 );
23
24## "constants"
25
26# 8 bits available
27# 128 is per-IP shitlist
28# 2 is IP hitlist
29# 1 not available so we don't $self->shoot(foot)
30our %bitfields = (
31 # block levels
32 0 => 16,
33 1 => 8,
34 2 => 4,
35 # ip
36 ip => 2,
37 # OOB
38 org => 32,
39 block => 64,
40 # "I'm a total spamming moron!" - per-IP only!
41 slist => 128
42);
43
44# probably needs some tuning; even 7 hits in a /24 is a pretty small percentage
45# number of IPs in a block of the given masklength needed to have that block automatically listed
46# defaults: (overridden by entries in db:autolist)
47our %autolist = (
48 31 => 1,
49 30 => 1,
50 29 => 2,
51 28 => 3,
52 27 => 4,
53 26 => 5,
54 25 => 6,
55 24 => 7,
56 23 => 8,
57 22 => 10,
58 21 => 13,
59 20 => 16,
60 19 => 19,
61 18 => 22,
62 17 => 26,
63 16 => 30,
64 15 => 34,
65 14 => 38,
66 13 => 42,
67 12 => 46,
68 11 => 50,
69 10 => 54,
70 9 => 58,
71 8 => 62,
72 7 => 2**31,
73 6 => 2**31,
74 5 => 2**31,
75 4 => 2**31,
76 3 => 2**31,
77 2 => 2**31,
78 1 => 2**31,
79 0 => 2**31
80);
81
82# le sigh. constants for masklength iterationing
83our @howmany = (1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2);
84
85# variables
86our $dbh;
87
88our $err;
89our $errstr;
90
91# basic object subs
92sub new {
93# iff we want to start taking arguments, or doing other things on instantiation
94# my $self = {};
95# bless $self, "DNSBL";
96# return $self;
97 bless {};
98}
99
100sub DESTROY {
101 my $self = shift;
102 $self->dbclose();
103}
104
105# JIC someone wants to close the db but not finish the script
106sub dbclose {
107 $dbh->rollback;
108 $dbh->disconnect;
109}
110
111## specific object subs:
112
113sub connect {
114 my $DSN = "DBI:Pg:host=dbhost;dbname=dnsbl";
115# my $DSN = "DBI:Pg:dbname=dnsbl";
116 my $user = "dnsbl";
117 my $pass = "spambgone";
118 ## want to NOT autocommit everything, it's unlikely we'll step on our own toes but...
119 $dbh = DBI->connect($DSN, $user, $pass, {
120 AutoCommit => 0,
121 PrintError => 1
122 })
123 or die "database inaccessible: ".$DBI::errstr;
124 my $sth = $dbh->prepare("SELECT masklen,ipcount FROM autolist");
125 $sth->execute;
126 while (my ($masklen,$ipcount) = $sth->fetchrow_array) {
127 $autolist{$masklen} = $ipcount;
128 }
129 return $dbh;
130}
131
132
133## DNSBLDB::initexport()
134# Prepare a couple of statement handles for later processing in export(). Assists in ~3x speed increase.
135my $parsth;
136my $sthmoron;
137sub initexport {
138 $parsth = $dbh->prepare("SELECT count(i.ip),b.block,b.level,b.listme AS oobblock,o.listme AS ooborg ".
139 "FROM iplist i INNER JOIN blocks b ON i.ip << b.block INNER JOIN orgs o ON b.orgid = o.orgid ".
140 "WHERE b.block >>= ? ".
141 "GROUP BY b.block,b.level,b.listme,o.listme ORDER BY b.block");
142 $sthmoron = $dbh->prepare("SELECT ip,s4list FROM iplist WHERE ip << ? ORDER BY ip");
143}
144
145
146## DNSBL::ipexists()
147# return report count if the IP has been reported, otherwise return undef
148sub ipexists {
149 my $self = shift;
150 my $ip = shift;
151 my $sth = $dbh->prepare("SELECT count FROM iplist WHERE ip=?");
152 $sth->execute($ip);
153 my ($ret) = $sth->fetchrow_array();
154 return $ret;
155} # end ipexists()
156
157
158# report an IP or URI to the db
159# increments a hit counter iff the reported IP or URI exists, otherwise it adds it
160sub report {
161 my $self = shift;
162 my $rep = shift;
163 my $sth;
164 my $rows = 0;
165 if ($rep =~ /^[\d.]+$/) {
166 # weesa gonna ASS-U-ME IP addresses are sanely formatted.
167 $sth = $dbh->prepare("SELECT count FROM iplist WHERE ip=?");
168 $sth->execute($rep) or die "eep? ".$dbh->errstr."\n";
169 $rows = $sth->rows;
170 if ($rows == 0) {
171 $sth = $dbh->prepare("INSERT INTO iplist (ip) VALUES (?)");
172 } elsif ($rows == 1) {
173 $sth = $dbh->prepare("UPDATE iplist SET count=count+1 WHERE ip=?");
174 } else {
175 die "db corrupt: found $rows matches on $rep\n";
176 }
177 $sth->execute($rep) or die "couldn't update listing for $rep: ".$dbh->errstr."\n";
178 } else {
179 return;
180 }
181 $dbh->commit;
182 return $rows;
183} # end report()
184
185
186# add a new org
187# return the orgid
188# if the org exists, return the orgid anyway
189sub addorg {
190 my $self = shift;
191 my $orgname = shift;
192 my $listme = shift || 'n';
193 my $ret = $self->orgexists($orgname);
194 return $ret if $ret;
195 my $sth = $dbh->prepare("INSERT INTO orgs (orgname,listme) VALUES (?,?)");
196 $sth->execute($orgname,$listme) or die "couldn't add org $orgname: ".$dbh->errstr."\n";
197 $dbh->commit;
198 $sth = $dbh->prepare("SELECT orgid FROM orgs WHERE orgname=?");
199 $sth->execute($orgname);
200 my ($orgid) = $sth->fetchrow_array();
201 return $orgid;
202} # end addorg
203
204
205# checks for existence - nb, exact match! No way to really handle anything else. :/
206sub orgexists {
207 my $self = shift;
208 my $org = shift;
209 my $sth = $dbh->prepare("SELECT orgid FROM orgs WHERE orgname=?");
210 $sth->execute($org);
211 my ($ret) = $sth->fetchrow_array();
212 return $ret;
213} # end orgexists();
214
215
216# add a block. requires the orgid
217##fixme needs error handling
218sub addblock {
219 my $self = shift;
220 my $blockin = shift;
221 my $orgid = shift;
222 my $level = shift;
223 $blockin =~ s/^\s+//;
224 $blockin =~ s/\s+$//;
225 my $block = new NetAddr::IP "$blockin"; # need this to clean up messes like ranges. sigh.
226
227 return "$blockin not a single CIDR range" if !$block;
228
229 local $dbh->{AutoCommit} = 1; # force autocommit
230
231 my $sth = $dbh->prepare("INSERT INTO blocks (block,orgid,level) VALUES (?,?,?)");
232 $sth->execute("$block",$orgid,$level);
233 return $sth->errstr if $sth->err;
234 # nb: no need to return anything, since the CIDR block is the key
235}
236
237
238sub blockexists {
239 my $self = shift;
240 my $block = shift;
241 my $sth = $dbh->prepare("SELECT count(*) FROM blocks WHERE block=?");
242 $sth->execute($block);
243 my ($ret) = $sth->fetchrow_array();
244 return $ret;
245}
246
247
248# returns list (block,orgname) for the block that contains the passed IP.
249# accepts a level argument if you don't want the top-level registrar allocation block
250sub getcontainer {
251 my $self = shift;
252 my $ip = shift;
253 my $level = shift || 0;
254 my $sth = $dbh->prepare("SELECT b.block,o.orgname FROM blocks b INNER JOIN orgs o ".
255 "ON b.orgid=o.orgid WHERE b.block >> ? AND b.level = ?");
256 $sth->execute($ip,$level);
257 return $sth->fetchrow_array();
258} # end getcontainer()
259
260
261# whee! Recursion is Fun!
262# Call ourself to dig down through the layers of blocks from registar-allocation
263# (level 0) to final block (level n, not to exceed 2)
264# Take a reference to a hash, and stuff it full of blacklisting goodness.
265# Optionally accept a level, block-container, and OOB block and org arguments for
266# the container to check and return
267# Returns no value directly
268# Calls itself to walk down the tree of containers
269sub export {
270 my $self = shift;
271 my $listhosts = shift;
272
273# Export data as CIDR netblocks or classful (A/B/C) blocks
274# Assume classful as it's more compatible with different DNS servers
275 my $mode = shift || 'class';
276
277# Assume we're checking the whole enchilada if we don't get told where to look.
278 my $level = shift || 0;
279 my $container = shift || '0.0.0.0/0';
280 my $bitmask = shift || 0;
281
282 if ($level > 3) {
283 warn "getting too deep, breaking off! ($container, $level)\n";
284 return;
285 }
286
287# fiddle $container into a sane state.
288 if ($container =~ m|^\d+\.\d+\.\d+/\d+$|) {
289 $container =~ s|/(\d+)$|.0/$1|;
290 } elsif ($container =~ m|^\d+\.\d+/\d+$|) {
291 $container =~ s|/(\d+)$|.0.0/$1|;
292 } elsif ($container =~ m|^\d+/(\d+)$|) {
293 $container =~ s|/(\d+)$|.0.0.0/$1|;
294 }
295
296 my $sth = $dbh->prepare("SELECT count(*) FROM blocks WHERE block << ?");
297 $sth->execute($container);
298 my ($nblocks) = $sth->fetchrow_array();
299
300 # need this for a bunch of things, may as well do it here
301 my ($masklen) = ($container =~ m|/(\d+)$|);
302
303# Update the bitmask variable with the current block info as needed.
304# Much faster than retrieving this data later (~3x faster!).
305 my $listme;
306 my $listorg;
307 my $bcount;
308 if ($container ne '0.0.0.0/0') {
309 $sth = $dbh->prepare("SELECT count(*) FROM iplist WHERE ip << ?");
310 $sth->execute($container);
311 ($bcount) = $sth->fetchrow_array();
312
313 $sth = $dbh->prepare("SELECT b.listme,o.listme ".
314 "FROM blocks b INNER JOIN orgs o ON b.orgid=o.orgid ".
315 "WHERE b.block = ?");
316 $sth->execute($container);
317 ($listme,$listorg) = $sth->fetchrow_array();
318
319 $bitmask |= $bitfields{$level-1} if $bcount >= $autolist{$masklen};
320 $bitmask |= $bitfields{block} if $listme;
321 $bitmask |= $bitfields{org} if $listorg;
322 }
323
324# hm. can't seem to move this prepare elsewhere. :(
325 if ($nblocks > 0) {
326 my $sthsubblocks = $dbh->prepare("SELECT block FROM blocks ".
327 "WHERE level=? and block << ? ORDER BY block, masklen(block) DESC");
328 $sthsubblocks->execute($level, $container);
329 while (my ($cidr) = $sthsubblocks->fetchrow_array()) {
330 $self->export($listhosts,$mode,$level+1,$cidr,$bitmask);
331 }
332 } # avoid checking content of subs if we don't have any
333
334 # don't check all 4.2 billion IPs individually if we're looking at all teh Intarwebs
335 return if $container eq '0.0.0.0/0';
336
337##fixme: need a way to dig out orphan IPs at all levels - IPs not found in a
338# subblock of the current container when the current container *has* subblocks
339# NB: this may be better handled as an out-of-band data-integrity-checker
340
341 # decrement level here so the right bitfield setting gets picked. this segment
342 # is inherently off-by-one from the block-recursion loop, and I can't see a
343 # better way to work around that. >:(
344 $level--;
345
346 if ($mode eq 'cidr') {
347 $listhosts->{$container} |= $bitmask if $bitmask && ($listme || $listorg || ($bcount >= $autolist{$masklen}));
348 } else {
349 # if $cidr->masklen is <= 24, iterate on /24 boundaries for bulk sublisting
350 # if $cidr->masklen is <= 16, iterate on /16 boundaries for bulk sublisting
351 # if $cidr->masklen is <= 8, iterate on /8 boundaries for bulk sublisting
352
353 if ($bitmask) {
354 my @blocksubs;
355 if ($masklen <= 30 && $masklen > 24) {
356 my ($net,$octet) = ($container =~ m|^(\d+\.\d+\.\d+\.)(\d+)/|);
357 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
358 my $host = "$net$entry";
359 $listhosts->{$host} = 0 if !defined($listhosts->{$host});
360 $listhosts->{$host} |= $bitmask;
361 }
362 } elsif ($masklen <= 24 && $masklen > 16) {
363 my ($net,$octet) = ($container =~ m|^(\d+\.\d+\.)(\d+)\.\d+/|);
364 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
365 my $twofour = "$net$entry.*";
366 $listhosts->{$twofour} |= $bitmask;
367 }
368 } elsif ($masklen <= 16 && $masklen > 8) {
369 my ($net,$octet) = ($container =~ m|^(\d+\.)(\d+)\.\d+\.\d+/|);
370 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
371 my $sixteen = "$net$entry.*";
372 $listhosts->{$sixteen} |= $bitmask;
373 }
374 } elsif ($masklen <= 8) {
375 my ($octet) = ($container =~ m|^(\d+)\.\d+\.\d+\.\d+/|);
376 for (my $entry = $octet; $entry < ($octet + $howmany[$masklen]); $entry++) {
377 my $eight = "$entry.*";
378 $listhosts->{$eight} |= $bitmask;
379 }
380 }
381
382 } # generate autolist entries for ips/octets not (yet) seen in reports
383
384 } # cidr vs classful mode
385
386 $sthmoron->execute($container);
387 while (my ($ip,$moron) = $sthmoron->fetchrow_array()) {
388 $listhosts->{$ip} |= $bitmask;
389 if ($moron) {
390 $listhosts->{$ip} = $bitfields{slist};
391 } else {
392 $listhosts->{$ip} |= $bitfields{ip};
393 }
394 }
395
396# get IPs which for reasons unknown are apparently allocated directly from the
397# parent registry (and so do not have containing netblocks in this system) O_o
398# select * from iplist where not (select count(*) from blocks where ip << block) > 0;
399
400 return;
401} # end export()
402
403
404sub export_alt {
405 my $self = shift;
406 my $listhosts = shift;
407 my $level = shift || 0;
408 my $container = shift || '0.0.0.0/0';
409 my $oobblock = shift || 0;
410 my $ooborg = shift || 0;
411
412#print "\nDEBUG: called with $level, $container, $oobblock, $ooborg\n";
413# if $level > 2 or $container =~ /^64\.76\./;
414# my %listhosts;
415
416# $level = 0 if !$level;
417 if ($level > 3) {
418 warn "getting too deep, breaking off!\n";
419 return;
420 }
421
422 my $sth = $dbh->prepare("select ip,s4list from iplist order by ip");
423 my $bsth = $dbh->prepare("select b.block,b.listme,b.level,o.listme ".
424 "from blocks b inner join orgs o on b.orgid=o.orgid ".
425 "where b.block >> ?");
426 while (my ($ip,$s4list) = $sth->fetchrow_array) {
427 $bsth->execute($ip);
428 while (my ($block,$blisted,$blevel,$olisted) = $bsth->fetchrow_array) {
429 $listhosts->{$ip} |= 0;
430 }
431 }
432
433} # end export_alt()
434
435
436## DNSBL::autolist_block()
437# check if a block should be autolisted
438sub autolist_block {
439 my $self = shift;
440 my $block = shift;
441
442 my $cidr = new NetAddr::IP "$block";
443 my $sth = $dbh->prepare("select count(*) from iplist where ip << ?");
444 $sth->execute("$cidr");
445 my ($count) = $sth->fetchrow_array;
446
447 return 1 if $count >= $autolist{$cidr->masklen};
448 return 0;
449} # end autolist_block()
450
451
452# make Perl happy
4531;
Note: See TracBrowser for help on using the repository browser.