source: trunk/tiny-import.pl@ 360

Last change on this file since 360 was 360, checked in by Kris Deugau, 12 years ago

/trunk

Checkpoint on tiny-import.pl

  • Handle wildcards sanely in the records that usefully support them. (mostly)
  • Handle TXT records in reverse zones
  • Properly put AAAA records in forward zones, not reverse zones
  • Property svn:executable set to *
  • Property svn:keywords set to Date Rev Author Id
File size: 14.7 KB
Line 
1#!/usr/bin/perl
2# dnsadmin shell-based import tool for tinydns flatfiles
3##
4# $Id: tiny-import.pl 360 2012-07-09 21:51:58Z kdeugau $
5# Copyright 2012 Kris Deugau <kdeugau@deepnet.cx>
6#
7# This program is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program. If not, see <http://www.gnu.org/licenses/>.
19##
20
21# WARNING: This is NOT a heavy-duty validator; it is assumed that the data
22# being imported is more or less sane. Only minor structural validation will
23# be done to weed out the most broken records.
24
25use strict;
26use warnings;
27
28use lib '.';
29use DNSDB qw(:ALL);
30
31if (!loadConfig()) {
32 warn "Using default configuration; unable to load custom settings: $DNSDB::errstr";
33}
34
35my $code;
36my ($dbh,$msg) = connectDB($config{dbname}, $config{dbuser}, $config{dbpass}, $config{dbhost});
37initGlobals($dbh) if $dbh;
38
39$dbh->{AutoCommit} = 0;
40$dbh->{RaiseError} = 1;
41
42my %cnt;
43my @deferred;
44my $errstr = '';
45
46foreach my $file (@ARGV) {
47 eval {
48 import(file => $file);
49# import(file => $file, nosoa => 1);
50 $dbh->rollback;
51# $dbh->commit;
52 };
53 if ($@) {
54 print "bleh: $@\n";
55die "die harder: $errstr\n";
56 }
57}
58
59 foreach (keys %cnt) {
60 print " $_ $cnt{$_}\n";
61 }
62
63exit 0;
64
65sub import {
66 our %args = @_;
67 my $flatfile = $args{file};
68 open FLAT, "<$flatfile";
69
70 our $recsth = $dbh->prepare("INSERT INTO records (domain_id,rdns_id,host,type,val,distance,weight,port,ttl) ".
71 " VALUES (?,?,?,?,?,?,?,?,?)");
72
73 my %deleg;
74
75 while (<FLAT>) {
76 next if /^#/;
77 next if /^\s*$/;
78 chomp;
79 recslurp($_);
80 }
81
82 # Try the deferred records again, once.
83 foreach (@deferred) {
84 # print "trying $_ again\n";
85 recslurp($_, 1);
86 }
87
88print scalar(@deferred)." deferred records in $flatfile\n";
89
90 # Sub for various nonstandard types with lots of pure bytes expressed in octal
91 # Takes a tinydns rdata string and count, returns a list of $count bytes as well
92 # as trimming those logical bytes off the front of the rdata string.
93 sub _byteparse {
94 my $src = shift;
95 my $count = shift;
96 my @ret;
97 for (my $i = 0; $i < $count; $i++) {
98 if ($$src =~ /^\\/) {
99 # we should have an octal bit
100 my ($tmp) = ($$src =~ /^(\\\d{3})/);
101 $tmp =~ s/\\/0/;
102 push @ret, oct($tmp);
103 $$src =~ s/^\\\d{3}//;
104 } else {
105 # we seem to have a byte expressed as an ASCII character
106 my ($tmp) = ($$src =~ /^(.)/);
107 push @ret, ord($tmp);
108 $$src =~ s/^.//;
109 }
110 }
111 return @ret;
112 }
113
114 # Convert octal-coded bytes back to something resembling normal characters, general case
115 sub _deoctal {
116 my $targ = shift;
117 while ($$targ =~ /\\(\d{3})/) {
118 my $sub = chr(oct($1));
119 $$targ =~ s/\\$1/$sub/g;
120 }
121 }
122
123 sub recslurp {
124 my $rec = shift;
125 my $nodefer = shift || 0;
126
127 $errstr = $rec; # this way at least we have some idea what went <splat>
128
129 if ($rec =~ /^=/) {
130 $cnt{APTR}++;
131
132##fixme: do checks like this for all types
133 if ($rec !~ /^=(?:\*|\\052)?[a-z0-9\._-]+:[\d\.]+:\d*/i) {
134 print "bad A+PTR $rec\n";
135 return;
136 }
137 my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
138 $host =~ s/^=//;
139 $host =~ s/\.$//;
140 $ttl = 0 if !$ttl;
141 $stamp = '' if !$stamp;
142 $loc = '' if !$loc;
143 $loc = '' if $loc =~ /^:+$/;
144 my $fparent = DNSDB::_hostparent($dbh, $host);
145 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($ip));
146 if ($fparent && $rparent) {
147 $recsth->execute($fparent, $rparent, $host, 65280, $ip, 0, 0, 0, $ttl);
148 } else {
149 push @deferred, $rec unless $nodefer;
150 # print "$tmporig deferred; can't find both forward and reverse zone parents\n";
151 }
152
153 } elsif ($rec =~ /^C/) {
154 $cnt{CNAME}++;
155
156 my ($host,$targ,$ttl,$stamp,$loc) = split /:/, $rec, 5;
157 $host =~ s/^C//;
158 $host =~ s/\.$//;
159 $host =~ s/^\\052/*/;
160 $ttl = 0 if !$ttl;
161 $stamp = '' if !$stamp;
162 $loc = '' if !$loc;
163 $loc = '' if $loc =~ /^:+$/;
164 if ($host =~ /\.arpa$/) {
165 ($code,$msg) = DNSDB::_zone2cidr($host);
166 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
167 $recsth->execute(0, $rparent, $targ, 5, $msg->addr, 0, 0, 0, $ttl);
168
169##fixme: automagically convert manually maintained sub-/24 delegations
170# my ($subip, $zone) = split /\./, $targ, 2;
171# ($code, $msg) = DNSDB::_zone2cidr($zone);
172# push @{$deleg{"$msg"}{iplist}}, $subip;
173#print "$msg $subip\n";
174
175 } else {
176 my $fparent = DNSDB::_hostparent($dbh, $host);
177 if ($fparent) {
178 $recsth->execute($fparent, 0, $host, 5, $targ, 0, 0, 0, $ttl);
179 } else {
180 push @deferred, $rec unless $nodefer;
181 # print "$tmporig deferred; can't find parent zone\n";
182 }
183 }
184
185 } elsif ($rec =~ /^\&/) {
186 $cnt{NS}++;
187
188 my ($zone,$ip,$ns,$ttl,$stamp,$loc) = split /:/, $rec, 6;
189 $zone =~ s/^\&//;
190 $zone =~ s/\.$//;
191 $ns =~ s/\.$//;
192 $ns = "$ns.ns.$zone" if $ns !~ /\./;
193 $ttl = 0 if !$ttl;
194 $stamp = '' if !$stamp;
195 $loc = '' if !$loc;
196 $loc = '' if $loc =~ /^:+$/;
197 if ($zone =~ /\.arpa$/) {
198 ($code,$msg) = DNSDB::_zone2cidr($zone);
199 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >>= ?", undef, ("$msg"));
200##fixme, in concert with the CNAME check for same; automagically
201# create "delegate" record instead for subzone NSes: convert above to use = instead of >>=
202# ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"))
203# if !$rparent;
204 if ($rparent) {
205 $recsth->execute(0, $rparent, $ns, 2, $msg, 0, 0, 0, $ttl);
206 } else {
207 push @deferred, $rec unless $nodefer;
208 }
209 } else {
210 my $fparent = DNSDB::_hostparent($dbh, $zone);
211 if ($fparent) {
212 $recsth->execute($fparent, 0, $zone, 2, $ns, 0, 0, 0, $ttl);
213 $recsth->execute($fparent, 0, $ns, 2, $ip, 0, 0, 0, $ttl) if $ip;
214 } else {
215 push @deferred, $rec unless $nodefer;
216 }
217 }
218
219 } elsif ($rec =~ /^\^/) {
220 $cnt{PTR}++;
221
222 my ($rip,$host,$ttl,$stamp,$loc) = split /:/, $rec, 5;
223 $rip =~ s/^\^//;
224 $rip =~ s/\.$//;
225 $ttl = 0 if !$ttl;
226 $stamp = '' if !$stamp;
227 $loc = '' if !$loc;
228 $loc = '' if $loc =~ /^:+$/;
229 my $rparent;
230 if (my ($i, $z) = ($rip =~ /^(\d+)\.(\d+-(?:\d+\.){4}in-addr.arpa)$/) ) {
231 ($code,$msg) = DNSDB::_zone2cidr($z);
232 # Exact matches only, because we're in a sub-/24 delegation
233##fixme: flag the type of delegation (range, subnet-with-dash, subnet-with-slash)
234# somewhere so we can recover it on export. probably best to do that in the revzone data.
235 ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ("$msg"));
236 $z =~ s/^[\d-]+//;
237 ($code,$msg) = DNSDB::_zone2cidr("$i.$z"); # Get the actual IP and normalize
238 } else {
239 ($code,$msg) = DNSDB::_zone2cidr($rip);
240 ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"));
241 }
242 if ($rparent) {
243 $recsth->execute(0, $rparent, $host, 12, $msg->addr, 0, 0, 0, $ttl);
244 } else {
245 push @deferred, $rec unless $nodefer;
246 }
247
248 } elsif ($rec =~ /^\+/) {
249 $cnt{A}++;
250
251 my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
252 $host =~ s/^\+//;
253 $host =~ s/\.$//;
254 $host =~ s/^\\052/*/;
255 $ttl = 0 if !$ttl;
256 $stamp = '' if !$stamp;
257 $loc = '' if !$loc;
258 $loc = '' if $loc =~ /^:+$/;
259
260 my $domid = DNSDB::_hostparent($dbh, $host);
261 if ($domid) {
262 $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl);
263 } else {
264 push @deferred, $rec unless $nodefer;
265 }
266
267 } elsif ($rec =~ /^Z/) {
268 $cnt{SOA}++;
269
270 my ($zone,$master,$contact,$serial,$refresh,$retry,$expire,$minttl,$ttl,$stamp,$loc) = split /:/, $rec, 11;
271 $zone =~ s/^Z//;
272 $zone =~ s/\.$//;
273 $master =~ s/\.$//;
274 $contact =~ s/\.$//;
275 $ttl = 0 if !$ttl;
276 $stamp = '' if !$stamp;
277 $loc = '' if !$loc;
278 $loc = '' if $loc =~ /^:+$/;
279 if ($zone =~ /\.arpa$/) {
280 ($code,$msg) = DNSDB::_zone2cidr($zone);
281 $dbh->do("INSERT INTO revzones (revnet,group_id,status) VALUES (?,1,1)", undef, ($msg));
282 my ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
283 $recsth->execute(0, $rdns, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl);
284 } else {
285 $dbh->do("INSERT INTO domains (domain,group_id,status) VALUES (?,1,1)", undef, ($zone));
286 my ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
287 $recsth->execute($domid, 0, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl);
288 }
289
290 } elsif ($rec =~ /^\@/) {
291 $cnt{MX}++;
292
293 my ($zone,$ip,$host,$dist,$ttl,$stamp,$loc) = split /:/, $rec, 7;
294 $zone =~ s/^\@//;
295 $zone =~ s/\.$//;
296 $zone =~ s/^\\052/*/;
297 $host =~ s/\.$//;
298 $host = "$host.mx.$zone" if $host !~ /\./;
299 $ttl = 0 if !$ttl;
300 $stamp = '' if !$stamp;
301 $loc = '' if !$loc;
302 $loc = '' if $loc =~ /^:+$/;
303
304# note we don't check for reverse domains here, because MX records don't make any sense in reverse zones.
305# if this really ever becomes an issue for someone it can be expanded to handle those weirdos
306
307 # allow for subzone MXes, since it's perfectly legitimate to simply stuff it all in a single parent zone
308 my $domid = DNSDB::_hostparent($dbh, $zone);
309 if ($domid) {
310 $recsth->execute($domid, 0, $zone, 15, $host, $dist, 0, 0, $ttl);
311 $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl) if $ip;
312 } else {
313 push @deferred, $rec unless $nodefer;
314 }
315
316 } elsif ($rec =~ /^'/) {
317 $cnt{TXT}++;
318
319 my ($fqdn, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 5;
320 $fqdn =~ s/^'//;
321 $fqdn =~ s/^\\052/*/;
322 _deoctal(\$rdata);
323 $ttl = 0 if !$ttl;
324 $stamp = '' if !$stamp;
325 $loc = '' if !$loc;
326 $loc = '' if $loc =~ /^:+$/;
327
328 if ($fqdn =~ /\.arpa$/) {
329 ($code,$msg) = DNSDB::_zone2cidr($fqdn);
330 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
331 $recsth->execute(0, $rparent, $rdata, 16, "$msg", 0, 0, 0, $ttl);
332 } else {
333 my $domid = DNSDB::_hostparent($dbh, $fqdn);
334 if ($domid) {
335 $recsth->execute($domid, 0, $fqdn, 16, $rdata, 0, 0, 0, $ttl);
336 } else {
337 push @deferred, $rec unless $nodefer;
338 }
339 }
340
341 } elsif ($rec =~ /^\./) {
342 $cnt{NSASOA}++;
343
344 my ($fqdn, $ip, $ns, $ttl, $stamp, $loc) = split /:/, $rec, 6;
345 $fqdn =~ s/^\.//;
346 $fqdn =~ s/\.$//;
347 $ns =~ s/\.$//;
348 $ns = "$ns.ns.$fqdn" if $ns !~ /\./;
349 $ttl = 0 if !$ttl;
350 $stamp = '' if !$stamp;
351 $loc = '' if !$loc;
352 $loc = '' if $loc =~ /^:+$/;
353
354 if ($fqdn =~ /\.arpa$/) {
355 ($code,$msg) = DNSDB::_zone2cidr($fqdn);
356 my ($rdns) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ($msg));
357 if (!$rdns) {
358 $errstr = "adding revzone $msg";
359 $dbh->do("INSERT INTO revzones (revnet,group_id,status) VALUES (?,1,1)", undef, ($msg));
360 ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
361# this would probably make a lot more sense to do hostmaster.$config{admindomain}
362 $recsth->execute(0, $rdns, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, "2560");
363 }
364 $recsth->execute(0, $rdns, $ns, 2, "$msg", 0, 0, 0, $ttl);
365##fixme: (?) implement full conversion of tinydns . records?
366# -> problem: A record for NS must be added to the appropriate *forward* zone, not the reverse
367#$recsth->execute(0, $rdns, $ns, 1, $ip, 0, 0, 0, $ttl)
368# ... auto-A-record simply does not make sense in reverse zones. Functionally
369# I think it would work, sort of, but it's a nasty mess and anyone hosting reverse
370# zones has names for their nameservers already.
371# Even the auto-nameserver-fqdn comes out... ugly.
372
373 } else {
374 my ($domid) = $dbh->selectrow_array("SELECT domain_id FROM domains WHERE lower(domain) = lower(?)",
375 undef, ($fqdn));
376 if (!$domid) {
377 $errstr = "adding domain $fqdn";
378 $dbh->do("INSERT INTO domains (domain,group_id,status) VALUES (?,1,1)", undef, ($fqdn));
379 ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
380 $recsth->execute($domid, 0, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, "2560");
381 }
382 $recsth->execute($domid, 0, $fqdn, 2, $ns, 0, 0, 0, $ttl);
383 $recsth->execute($domid, 0, $ns, 1, $ip, 0, 0, 0, $ttl) if $ip;
384 }
385
386
387 } elsif ($rec =~ /^\%/) {
388 $cnt{VIEWS}++;
389
390 } elsif ($rec =~ /^:/) {
391 $cnt{NCUST}++;
392# Big section. Since tinydns can publish anything you can encode properly, but only provides official
393# recognition and handling for the core common types, this must deal with the leftovers.
394# :fqdn:type:rdata:ttl:time:loc
395
396 my (undef, $fqdn, $type, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 7;
397 $fqdn =~ s/\.$//;
398 $fqdn =~ s/^\\052/*/;
399 $ttl = 0 if !$ttl;
400 $stamp = '' if !$stamp;
401 $loc = '' if !$loc;
402 $loc = '' if $loc =~ /^:+$/;
403
404 if ($type == 33) {
405 # SRV
406 my ($prio, $weight, $port, $target) = (0,0,0,0);
407
408 my @tmp = _byteparse(\$rdata, 2);
409 $prio = $tmp[0] * 256 + $tmp[1];
410 @tmp = _byteparse(\$rdata, 2);
411 $weight = $tmp[0] * 256 + $tmp[1];
412 @tmp = _byteparse(\$rdata, 2);
413 $port = $tmp[0] * 256 + $tmp[1];
414
415 $rdata =~ s/\\\d{3}/./g;
416 ($target) = ($rdata =~ /^\.(.+)\.$/);
417# hmm. the above *should* work, but What If(TM) we have ASCII-range bytes
418# representing the target's fqdn part length(s)? axfr-get doesn't seem to,
419# probably because dec. 33->63 includes most punctuation and all the numbers
420# while ($rdata =~ /(\\\d{3})/) {
421# my $cnt = $1;
422# $rdata =~ s/^$cnt//;
423# $cnt =~ s/^\\/0/;
424# $cnt = oct($cnt);
425# my ($seg) = ($rdata =~ /^(.{$cnt})/);
426# $target .=
427# }
428
429 my $domid = DNSDB::_hostparent($dbh, $fqdn);
430 if ($domid) {
431 $recsth->execute($domid, 0, $fqdn, 33, $target, $prio, $weight, $port, $ttl) if $domid;
432 } else {
433 push @deferred, $rec unless $nodefer;
434 }
435
436 } elsif ($type == 28) {
437 # AAAA
438 my @v6;
439
440 for (my $i=0; $i < 8; $i++) {
441 my @tmp = _byteparse(\$rdata, 2);
442 push @v6, sprintf("%0.4x", $tmp[0] * 256 + $tmp[1]);
443 }
444 my $val = NetAddr::IP->new(join(':', @v6));
445
446 my $fparent = DNSDB::_hostparent($dbh, $fqdn);
447 if ($fparent) {
448 $recsth->execute($fparent, 0, $fqdn, 28, $val->addr, 0, 0, 0, $ttl);
449 } else {
450 push @deferred, $rec unless $nodefer;
451 }
452
453 } else {
454 # ... uhhh, dunno
455 }
456
457 } else {
458 $cnt{other}++;
459 print " $_\n";
460 }
461 }
462
463 close FLAT;
464}
Note: See TracBrowser for help on using the repository browser.