source: trunk/dnsbl/setparents.pl@ 32

Last change on this file since 32 was 32, checked in by Kris Deugau, 13 years ago

/trunk/dnsbl

Changeset to improve export speed on DNSBL database. By keeping
IP counts for each block, and directly tracking the parent of
each block and IP, an export taking ~90 seconds can be brought
down under 20.

  • updated initial tabledef SQL
  • Add addparents.sql SQL tabeldef and setparents.pl data update script
  • Tweak reporting of IP since it now requires the parent block(s) so the parent field can be filled in

Also add delist-ip script and Makefile

  • Property svn:executable set to *
  • Property svn:keywords set to Date Rev Author Id
File size: 2.1 KB
Line 
1#!/usr/bin/perl
2# hack to set new parent column on blocks and iplist tables
3
4use strict;
5use warnings;
6use DBI;
7
8use DNSBL;
9
10my $dnsbl = new DNSBL;
11
12# default DB info - all other settings should be loaded from the DB.
13my $dbhost = "dbhost";
14my $dbname = "dnsbl";
15my $dbuser = "dnsbl";
16my $dbpass = "spambgone";
17
18die "Need config argument\n" if !$ARGV[0];
19my $cfgname = shift @ARGV;
20
21# Load a config ref containing DB host, name, user, and pass info based on
22# from the server name + full script web path. This allows us to host
23# multiple instances without having to duplicate the code.
24# This file is a Perl fragment to be processed inline.
25if (-e "/etc/dnsbl/$cfgname.conf") {
26 my $cfg = `cat /etc/dnsbl/$cfgname.conf`;
27 ($cfg) = ($cfg =~ /^(.+)$/s); # avoid warnings, failures, and general nastiness with taint mode
28 eval $cfg;
29}
30
31my $dbh = $dnsbl->connect($dbhost, $dbname, $dbuser, $dbpass);
32
33print "block parents\n";
34my $sthplist = $dbh->prepare("SELECT block FROM blocks WHERE level=?");
35my $sthclist = $dbh->prepare("UPDATE blocks SET parent=? WHERE level=? AND block << ?");
36foreach my $level ((0,1)) {
37 $sthplist->execute($level) or die "failed to get blocklist: ".$sthplist->errstr."\n";
38 while (my ($block) = $sthplist->fetchrow_array) {
39 $sthclist->execute($block,$level+1,$block) or die "failed setparent on $block subs: ".$sthclist->errstr."\n";
40 }
41}
42print " done\n";
43
44print "block counts\n";
45my $sthblist = $dbh->prepare("SELECT block FROM blocks");
46my $sthcount = $dbh->prepare("UPDATE blocks SET ipcount=(SELECT count(*) FROM iplist WHERE ip << ?) WHERE block = ?");
47$sthblist->execute;
48while (my ($block) = $sthblist->fetchrow_array) {
49 $sthcount->execute($block,$block) or die "failed setcount on $block: ".$sthcount->errstr."\n";
50}
51print " done\n";
52
53print "ip parents\n";
54$sthplist = $dbh->prepare("SELECT block FROM blocks ORDER BY level");
55$sthclist = $dbh->prepare("UPDATE iplist SET parent = ? WHERE ip << ?");
56$sthplist->execute;
57while (my ($block) = $sthplist->fetchrow_array) {
58 $sthclist->execute($block,$block);
59}
60
61# and yes, we do this in one huge block.
62$dbh->commit;
63print " done\n";
Note: See TracBrowser for help on using the repository browser.