source: trunk/dnsbl/export-dnsbl@ 32

Last change on this file since 32 was 32, checked in by Kris Deugau, 13 years ago

/trunk/dnsbl

Changeset to improve export speed on DNSBL database. By keeping
IP counts for each block, and directly tracking the parent of
each block and IP, an export taking ~90 seconds can be brought
down under 20.

  • updated initial tabledef SQL
  • Add addparents.sql SQL tabeldef and setparents.pl data update script
  • Tweak reporting of IP since it now requires the parent block(s) so the parent field can be filled in

Also add delist-ip script and Makefile

  • Property svn:executable set to *
  • Property svn:keywords set to Date Rev Author Id
File size: 3.1 KB
Line 
1#!/usr/bin/perl
2# Export DNSBL data
3# 2009/05/26 kdeugau@deepnet.cx
4# need to stub it out so it can use a module to actually write zone
5# data for different NS software
6# completed 2008/08/14
7
8use strict;
9use warnings;
10use DBI;
11
12use DNSBL;
13
14my $dnsbl = new DNSBL;
15
16# default DB info - all other settings should be loaded from the DB.
17my $dbhost = "localhost";
18my $dbname = "dnsbl";
19my $dbuser = "dnsbl";
20my $dbpass = "spambgone";
21
22die "Need config argument\n" if !$ARGV[0];
23my $cfgname = shift @ARGV;
24
25# Load a config ref containing DB host, name, user, and pass info based on
26# from the server name + full script web path. This allows us to host
27# multiple instances without having to duplicate the code.
28# This file is a Perl fragment to be processed inline.
29if (-e "/etc/dnsbl/$cfgname.conf") {
30 my $cfg = `cat /etc/dnsbl/$cfgname.conf`;
31 ($cfg) = ($cfg =~ /^(.+)$/s); # avoid warnings, failures, and general nastiness with taint mode
32 eval $cfg;
33}
34
35my $dbh = $dnsbl->connect($dbhost, $dbname, $dbuser, $dbpass);
36
37my %config;
38my $sth = $dbh->prepare("SELECT key,value FROM misc");
39$sth->execute;
40while (my ($key,$value) = $sth->fetchrow_array) {
41 $config{$key} = $value;
42}
43
44my %iplist;
45my $ipref = \%iplist;
46
47my $mode = $ARGV[0] || 'tiny';
48
49$dnsbl->initexport;
50#$dnsbl->export($ipref,$mode,1,'50.22.0.0/15');
51$dnsbl->export($ipref,$mode);
52
53##fixme - mode should pick actual output, not just export mode
54if ($mode eq 'cidr') {
55 # SOA, NS records. Maybe dnscache needs them?
56 print "\$SOA 900 ".($config{blzone} ? $config{blzone} : 'company').".dnsbl systems.company.com 0 1200 600 600 900\n".
57 "\$NS 3600 127.0.0.1\n".
58 "\$TTL 900\n";
59
60 # more or less raw CIDR block-and-IP info. rbldnsd format for convenience.
61 foreach (sort ipcmp keys %iplist) {
62 print "$_:127.0.0.$iplist{$_}:".
63 ($iplist{$_} & 2 ?
64 ($config{iplisted} ? $config{iplisted} : '$ relayed a reported spam') :
65 ($config{blocklisted} ? $config{blocklisted} : 'Netblock listed on one or more criteria')
66 )."\n";
67 }
68} else {
69 foreach (sort ipcmp keys %iplist) {
70 my ($o1,$o2,$o3,$o4) = (/^(\d+)\.([\d*]+)(?:\.([\d*]+)(?:\.([\d*]+))?)?$/);
71 print "+".(defined($o4) ? "$o4." : '').(defined($o3) ? "$o3." : '').(defined($o2) ? "$o2." : '').
72 "$o1.spamhosts.company.com:127.0.0.$iplist{$_}:900:::\n";
73 }
74}
75
76exit 0;
77
78# IP address comparison sub
79sub ipcmp {
80 my ($a1,$a2,$a3,$a4,$a5) = ($a =~ /^(\d+)\.([\d*]+)(?:\.([\d*]+)(?:\.([\d*]+))?)?(?:\/(\d+))?$/);
81 my ($b1,$b2,$b3,$b4,$b5) = ($b =~ /^(\d+)\.([\d*]+)(?:\.([\d*]+)(?:\.([\d*]+))?)?(?:\/(\d+))?$/);
82# le sigh. knew it wasn't going to be simple...
83 $b2 = -1 if $b2 && $b2 eq '*';
84 $b3 = -1 if $b3 && $b3 eq '*';
85 $b4 = -1 if $b4 && $b4 eq '*';
86 $b5 = 128 if !defined($b5);
87 $a2 = -1 if $a2 && $a2 eq '*';
88 $a3 = -1 if $a3 && $a3 eq '*';
89 $a4 = -1 if $a4 && $a4 eq '*';
90 $a5 = 128 if !defined($a5);
91 return 1 if $a1 > $b1;
92 return -1 if $a1 < $b1;
93 return 1 if $a2 > $b2;
94 return -1 if $a2 < $b2;
95 return 1 if $a3 > $b3;
96 return -1 if $a3 < $b3;
97 return 1 if $a4 > $b4;
98 return -1 if $a4 < $b4;
99 return 1 if $a5 > $b5;
100 return -1 if $a5 < $b5;
101 return 0;
102}
Note: See TracBrowser for help on using the repository browser.