source: trunk/tiny-import.pl@ 926

Last change on this file since 926 was 843, checked in by Kris Deugau, 4 years ago

/trunk

Minor accumulated refinements on tiny-import.pl

  • Update include path finder - see #80
  • Clean up extra trailing :'s that sometimes confused the location
  • Better handle "empty" TTL values
  • Slightly improve SOA serial type tagging; only handles the most common case of "let tinydns handle it" UNIX-epoch serials for now
  • Property svn:executable set to *
  • Property svn:keywords set to Date Rev Author Id
File size: 33.5 KB
Line 
1#!/usr/bin/perl
2# dnsadmin shell-based import tool for tinydns flatfiles
3##
4# $Id: tiny-import.pl 843 2022-04-29 19:45:16Z kdeugau $
5# Copyright 2012-2014,2020-2022 Kris Deugau <kdeugau@deepnet.cx>
6#
7# This program is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program. If not, see <http://www.gnu.org/licenses/>.
19##
20
21# WARNING: This is NOT a heavy-duty validator; it is assumed that the data
22# being imported is more or less sane. Only minor structural validation will
23# be done to weed out the most broken records.
24
25use strict;
26use warnings;
27use POSIX;
28use Time::TAI64 qw(:tai);
29
30# Taint-safe (ish) voodoo to push "the directory the script is in" into @INC.
31use File::Spec ();
32use File::Basename ();
33my $path;
34BEGIN {
35 $path = File::Basename::dirname(File::Spec->rel2abs($0));
36 if ($path =~ /(.*)/) {
37 $path = $1;
38 }
39}
40use lib $path;
41
42use DNSDB;
43
44my $dnsdb = new DNSDB;
45
46usage() if !@ARGV;
47
48my %importcfg = (
49 rw => 0,
50 conv => 0,
51 trial => 0,
52 legacy => 0,
53 merge => 0,
54 group => 1,
55 );
56my $gnum = '';
57# Handle some command-line arguments
58while ($ARGV[0] =~ /^-/) {
59 my $arg = shift @ARGV;
60 usage() if $arg !~ /^-(?:[rclmt]+|g\d*)$/;
61 # -r rewrite imported files to comment imported records
62 # -c coerce/downconvert A+PTR = records to PTR
63 # -l swallow A+PTR as-is
64 # -m merge PTR and A/AAAA as possible
65 # -t trial mode; don't commit to DB or actually rewrite flatfile (disables -r)
66 # -g import to specified group (name or ID) instead of group 1
67 $arg =~ s/^-//;
68# for Reasons (none clear), $arg is undefined yet defined, but only when number characters are involved. Ebbeh?
69no warnings qw(uninitialized);
70 if ($arg =~ /^g/) {
71 if ($arg eq 'g') {
72 $importcfg{group} = shift @ARGV;
73 } else {
74 $arg =~ s/^g//;
75 $importcfg{group} = $arg;
76 }
77 } else {
78 my @tmp = split //, $arg;
79 foreach (@tmp) {
80 $importcfg{rw} = 1 if $_ eq 'r';
81 $importcfg{conv} = 1 if $_ eq 'c';
82 $importcfg{legacy} = 1 if $_ eq 'l';
83 $importcfg{merge} = 1 if $_ eq 'm';
84 $importcfg{trial} = 1 if $_ eq 't';
85 }
86 }
87 use warnings qw(uninitialized);
88}
89$importcfg{rw} = 0 if $importcfg{trial};
90
91# allow group names
92if ($importcfg{group} =~ /^\d+$/) {
93 $importcfg{groupname} = $dnsdb->groupName($importcfg{group});
94} else {
95 $importcfg{groupname} = $importcfg{group};
96 $importcfg{group} = $dnsdb->groupID($importcfg{groupname});
97}
98
99die usage() if $importcfg{group} !~ /^\d+$/;
100
101sub usage {
102 die q(usage: tiny-import.pl [-rclt] [-gnn] [-g name] datafile1 datafile2 ... datafileN ...
103 -r Rewrite all specified data files with a warning header indicating the
104 records are now managed by web, and commenting out all imported records.
105 The directory containing any given datafile must be writable.
106 -c Convert any A+PTR (=) record to a bare PTR if the forward domain is
107 not present in the database. Note this does NOT look forward through
108 a single file, nor across multiple files handled in the same run.
109 Multiple passes may be necessary if SOA and = records are heavily
110 intermixed and not clustered together.
111 -l (for "legacy") Force import of A+PTR records as-is. Mutually exclusive
112 with -c. -l takes precedence as -c is lossy.
113 -m Merge PTR and A or AAAA records to A+PTR or AAAA+PTR records where possible
114 -gnnn or -g nnn or -g name
115 Import new zones into this group (group name or ID accepted) instead of
116 the root/default group 1
117 -t Trial run mode; spits out records that would be left unimported.
118 Disables -r if set.
119
120 -r and -c may be combined (-rc)
121
122 datafileN is any tinydns record data file.
123);
124}
125
126my $code;
127my $dbh = $dnsdb->{dbh};
128
129# collect some things for logging
130($dnsdb->{logusername}, undef, undef, undef, undef, undef, $dnsdb->{logfullname}) = getpwuid($<);
131$dnsdb->{logfullname} =~ s/,//g;
132$dnsdb->{loguserid} = 0; # not worth setting up a pseudouser the way the RPC system does
133$dnsdb->{logusername} = $dnsdb->{logusername}."/tiny-import.pl";
134$dnsdb->{logfullname} = $dnsdb->{logusername} if !$dnsdb->{logfullname};
135$dnsdb->{logfullname} = $dnsdb->{logfullname}."/tiny-import.pl";
136
137$dbh->{AutoCommit} = 0;
138$dbh->{RaiseError} = 1;
139
140my %cnt;
141my @deferred;
142my $converted = 0;
143my $errstr = '';
144
145foreach my $file (@ARGV) {
146 my %filecount;
147 my $logentry = "Import records from $file: ";
148 eval {
149 import(file => $file, cnt => \%filecount);
150 if (%filecount) {
151 foreach (sort keys %filecount) {
152 $logentry .= "$_ $filecount{$_}, ";
153 $cnt{$_} += $filecount{$_};
154 }
155 $logentry =~ s/[\s,]+$//;
156 $dnsdb->_log(group_id => $importcfg{group}, entry => $logentry);
157 }
158 $dbh->rollback if $importcfg{trial};
159 $dbh->commit unless $importcfg{trial};
160 };
161 if ($@) {
162 print "Failure trying to import $file: $@\n $errstr\n";
163 unlink ".$file.$$" if $importcfg{rw}; # cleanup
164 $dbh->rollback;
165 }
166}
167
168# print summary count of record types encountered
169foreach (sort keys %cnt) {
170 print " $_ $cnt{$_}\n";
171}
172
173exit 0;
174
175sub import {
176 our %args = @_;
177 my $flatfile = $args{file};
178 my $filecnt = $args{cnt};
179 my @fpath = split '/', $flatfile;
180 $fpath[$#fpath] = ".$fpath[$#fpath]";
181 my $rwfile = join('/', @fpath);#.".$$";
182
183 open FLAT, "<$flatfile";
184
185 if ($importcfg{rw}) {
186 open RWFLAT, ">$rwfile" or die "Couldn't open tempfile $rwfile for rewriting: $!\n";
187 print RWFLAT "# WARNING: Records in this file have been imported to the web UI.\n#\n";
188 }
189
190 our $recsth = $dbh->prepare("INSERT INTO records (domain_id,rdns_id,host,type,val,distance,weight,port,ttl,location,stamp,expires,stampactive) ".
191 " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)");
192
193 # for A/AAAA records
194 our $revcheck = $dbh->prepare("SELECT rdns_id,record_id,ttl FROM records WHERE host=? AND val=? AND type=12");
195 our $mergefwd = $dbh->prepare("UPDATE records SET type=?,domain_id=?,ttl=? WHERE record_id=?");
196 # for PTR records
197 our $fwdcheck = $dbh->prepare("SELECT domain_id,record_id,ttl FROM records WHERE host=? AND val=? AND (type=1 OR type=28)");
198 our $mergerev = $dbh->prepare("UPDATE records SET type=?,rdns_id=?,ttl=? WHERE record_id=?");
199
200 my %deleg;
201
202 my $ok = 0;
203 while (<FLAT>) {
204 if (/^#/ || /^\s*$/) {
205 print RWFLAT "#$_" if $importcfg{rw};
206 next;
207 }
208 chomp;
209 s/\s*$//;
210 s/:+$//;
211 my $recstat = recslurp($_, $filecnt);
212 $ok++ if $recstat;
213 if ($importcfg{rw}) {
214 if ($recstat) {
215 print RWFLAT "#$_\n";
216 } else {
217 print RWFLAT "$_\n";
218 }
219 }
220 }
221
222 # Move the rewritten flatfile in place of the original, so that any
223 # external export processing will pick up any remaining records.
224 if ($importcfg{rw}) {
225 close RWFLAT;
226 rename "$rwfile", $flatfile;
227 }
228
229 # Show the failed records
230 foreach (@deferred) {
231 print "failed to import $_\n";
232 }
233
234##fixme: hmm. can't write the record back to the flatfile in the
235# main while above, then come down here and import it anyway, can we?
236# # Try the deferred records again, once.
237# foreach (@deferred) {
238# print "trying $_ again\n";
239# recslurp($_, 1);
240# }
241
242 # .. but we can at least say how many records weren't imported.
243 print "$ok OK, ".scalar(@deferred)." deferred, $converted downconverted records in $flatfile\n";
244 undef @deferred;
245 $converted = 0;
246
247 # Sub for various nonstandard types with lots of pure bytes expressed in octal
248 # Takes a tinydns rdata string and count, returns a list of $count bytes as well
249 # as trimming those logical bytes off the front of the rdata string.
250 sub _byteparse {
251 my $src = shift;
252 my $count = shift;
253 my @ret;
254 for (my $i = 0; $i < $count; $i++) {
255 if ($$src =~ /^\\/) {
256 # we should have an octal bit
257 my ($tmp) = ($$src =~ /^(\\\d{3})/);
258 $tmp =~ s/\\/0/;
259 push @ret, oct($tmp);
260 $$src =~ s/^\\\d{3}//;
261 } else {
262 # we seem to have a byte expressed as an ASCII character
263 my ($tmp) = ($$src =~ /^(.)/);
264 push @ret, ord($tmp);
265 $$src =~ s/^.//;
266 }
267 }
268 return @ret;
269 }
270
271 # Convert octal-coded bytes back to something resembling normal characters, general case
272 sub _deoctal {
273 my $targ = shift;
274 while ($$targ =~ /\\(\d{3})/) {
275 my $sub = chr(oct($1));
276 $$targ =~ s/\\$1/$sub/g;
277 }
278 }
279
280 sub _rdata2string {
281 my $rdata = shift;
282 my $tmpout = '';
283 while ($rdata) {
284 my $bytecount = 0;
285 if ($rdata =~ /^\\/) {
286 ($bytecount) = ($rdata =~ /^(\\\d{3})/);
287 $bytecount =~ s/\\/0/;
288 $bytecount = oct($bytecount);
289 $rdata =~ s/^\\\d{3}//;
290 } else {
291 ($bytecount) = ($rdata =~ /^(.)/);
292 $bytecount = ord($bytecount);
293 $rdata =~ s/^.//;
294 }
295 my @tmp = _byteparse(\$rdata, $bytecount);
296 foreach (@tmp) { $tmpout .= chr($_); }
297##fixme: warn or fail on long (>256? >512? >321?) strings
298 }
299 return $tmpout;
300 }
301
302 sub _rdata2hex {
303 my $rdata = shift;
304 my $tmpout = '';
305 while ($rdata) {
306 my $byte = '';
307 if ($rdata =~ /^\\/) {
308 ($byte) = ($rdata =~ /^(\\\d{3})/);
309 $byte =~ s/\\/0/;
310 $tmpout .= sprintf("%0.2x", oct($byte));
311 $rdata =~ s/^\\\d{3}//;
312 } else {
313 ($byte) = ($rdata =~ /^(.)/);
314 $tmpout .= sprintf("%0.2x", ord($byte));
315 $rdata =~ s/^.//;
316 }
317 }
318 return $tmpout;
319 }
320
321 sub calcstamp {
322 my $stampin = shift;
323 my $ttl = shift;
324 my $pzone = shift;
325 my $revrec = shift;
326
327 return ($ttl, 'n', 'n', '1970-01-01 00:00:00 -0') if !$stampin;
328
329##fixme Yes, this fails for records in 2038 sometime. No, I'm not going to care for a while.
330 $stampin = "\@$stampin"; # Time::TAI64 needs the leading @. Feh.
331 my $u = tai2unix($stampin);
332 $stampin = strftime("%Y-%m-%d %H:%M:%S %z", localtime($u));
333 my $expires = 'n';
334 if ($ttl) {
335 # TTL can stay put.
336 } else {
337 # TTL on import is 0, almost certainly wrong. Get the parent zone's SOA and use the minttl.
338 my $soa = $dnsdb->getSOA('n', $revrec, $pzone);
339 $ttl = $soa->{minttl};
340 $expires = 'y';
341 }
342 return ($ttl, 'y', $expires, $stampin);
343 }
344
345 sub recslurp {
346 my $rec = shift;
347 my $filecnt = shift;
348 my $nodefer = shift || 0;
349 my $impok = 1;
350 my $msg;
351
352 $errstr = $rec; # this way at least we have some idea what went <splat>
353
354 if ($rec =~ /^=/) {
355 $filecnt->{'A+PTR'}++;
356
357##fixme: do checks like this for all types
358 if ($rec !~ /^=(?:\*|\\052)?[a-z0-9\._-]+:[\d\.]+:\d*/i) {
359 print "bad A+PTR $rec\n";
360 return;
361 }
362 my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
363 $host =~ s/^=//;
364 $host =~ s/\.$//;
365 $ttl = -1 if $ttl eq '';
366 $stamp = '' if !$stamp;
367 $loc = '' if !$loc;
368 $loc = '' if $loc =~ /^:+$/;
369 my $fparent = $dnsdb->_hostparent($host);
370 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($ip));
371
372 my $stampactive = 'n';
373 my $expires = 'n';
374
375 # can't set a timestamp on an orphaned record. we'll actually fail import of this record a little later.
376 if ($fparent || $rparent) {
377 if ($fparent) {
378 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
379 } else {
380 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
381 }
382 }
383
384 if ($fparent && $rparent) {
385 $recsth->execute($fparent, $rparent, $host, 65280, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
386 } else {
387 if ($importcfg{legacy}) {
388 # Just import it already! Record may still be subject to downconversion on editing.
389 $fparent = 0 if !$fparent;
390 $rparent = 0 if !$rparent;
391 if ($fparent || $rparent) {
392 $recsth->execute($fparent, $rparent, $host, 65280, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
393 } else {
394 # No parents found, cowardly refusing to add a dangling record
395 push @deferred, $rec unless $nodefer;
396 $impok = 0;
397 }
398 } elsif ($importcfg{conv}) {
399 # downconvert A+PTR if forward zone is not found
400 $recsth->execute(0, $rparent, $host, 12, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
401 $converted++;
402 } else {
403 push @deferred, $rec unless $nodefer;
404 $impok = 0;
405 # print "$tmporig deferred; can't find both forward and reverse zone parents\n";
406 }
407 }
408
409 } elsif ($rec =~ /^C/) {
410 $filecnt->{CNAME}++;
411
412 my ($host,$targ,$ttl,$stamp,$loc) = split /:/, $rec, 5;
413 $host =~ s/^C//;
414 $host =~ s/\.$//;
415 $host =~ s/^\\052/*/;
416 $ttl = -1 if !defined($ttl) || $ttl eq '';
417 $stamp = '' if !$stamp;
418 $loc = '' if !$loc;
419 $loc = '' if $loc =~ /^:+$/;
420
421 my $stampactive = 'n';
422 my $expires = 'n';
423
424 if ($host =~ /\.arpa$/) {
425 ($code,$msg) = DNSDB::_zone2cidr($host);
426 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
427 if ($rparent) {
428 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
429 $recsth->execute(0, $rparent, $targ, 5, $msg->addr, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
430 } else {
431 push @deferred, $rec unless $nodefer;
432 $impok = 0;
433 # print "$tmporig deferred; can't find parent zone\n";
434 }
435
436##fixme: automagically convert manually maintained sub-/24 delegations
437# my ($subip, $zone) = split /\./, $targ, 2;
438# ($code, $msg) = DNSDB::_zone2cidr($zone);
439# push @{$deleg{"$msg"}{iplist}}, $subip;
440#print "$msg $subip\n";
441
442 } else {
443 my $fparent = $dnsdb->_hostparent($host);
444 if ($fparent) {
445 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
446 $recsth->execute($fparent, 0, $host, 5, $targ, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
447 } else {
448 push @deferred, $rec unless $nodefer;
449 $impok = 0;
450 # print "$tmporig deferred; can't find parent zone\n";
451 }
452 }
453
454 } elsif ($rec =~ /^\&/) {
455 $filecnt->{NS}++;
456
457 my ($zone,$ip,$ns,$ttl,$stamp,$loc) = split /:/, $rec, 6;
458 $zone =~ s/^\&//;
459 $zone =~ s/\.$//;
460 $ns =~ s/\.$//;
461 $ns = "$ns.ns.$zone" if $ns !~ /\./;
462 $ttl = -1 if $ttl eq '';
463 $stamp = '' if !$stamp;
464 $loc = '' if !$loc;
465 $loc = '' if $loc =~ /^:+$/;
466
467 my $stampactive = 'n';
468 my $expires = 'n';
469
470 if ($zone =~ /\.arpa$/) {
471 ($code,$msg) = DNSDB::_zone2cidr($zone);
472 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >>= ?", undef, ("$msg"));
473##fixme, in concert with the CNAME check for same; automagically
474# create "delegate" record instead for subzone NSes: convert above to use = instead of >>=
475# ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"))
476# if !$rparent;
477 if ($rparent) {
478 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
479 $recsth->execute(0, $rparent, $ns, 2, $msg, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
480 } else {
481 push @deferred, $rec unless $nodefer;
482 $impok = 0;
483 }
484 } else {
485 my $fparent = $dnsdb->_hostparent($zone);
486 if ($fparent) {
487 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
488 $recsth->execute($fparent, 0, $zone, 2, $ns, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
489 $recsth->execute($fparent, 0, $ns, 2, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive) if $ip;
490 } else {
491 push @deferred, $rec unless $nodefer;
492 $impok = 0;
493 }
494 }
495
496 } elsif ($rec =~ /^\^/) {
497 $filecnt->{PTR}++;
498
499 my ($rip,$host,$ttl,$stamp,$loc) = split /:/, $rec, 5;
500 $rip =~ s/^\^//;
501 $rip =~ s/\.$//;
502 $ttl = -1 if $ttl eq '';
503 $stamp = '' if !$stamp;
504 $loc = '' if !$loc;
505 $loc = '' if $loc =~ /^:+$/;
506
507 my $stampactive = 'n';
508 my $expires = 'n';
509
510 my $rparent;
511 if (my ($i, $z) = ($rip =~ /^(\d+)\.(\d+-(?:\d+\.){4}in-addr.arpa)$/) ) {
512 ($code,$msg) = DNSDB::_zone2cidr($z);
513 # Exact matches only, because we're in a sub-/24 delegation
514##fixme: flag the type of delegation (range, subnet-with-dash, subnet-with-slash)
515# somewhere so we can recover it on export. probably best to do that in the revzone data.
516 ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ("$msg"));
517 $z =~ s/^[\d-]+//;
518 ($code,$msg) = DNSDB::_zone2cidr("$i.$z"); # Get the actual IP and normalize
519 } else {
520 ($code,$msg) = DNSDB::_zone2cidr($rip);
521 ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"));
522 }
523
524 if ($rparent) {
525##fixme: really want to pull this DB call inside an if $importcfg{merge},
526# but then we need to duplicate the insert for the case where the matching
527# reverse doesn't exist.
528 $host =~ s/\.$//g; # pure sytactic sugar, we don't store this trailing dot.
529 $fwdcheck->execute($host, $msg->addr);
530 my ($domid, $recid, $rttl) = $fwdcheck->fetchrow_array;
531 if ($importcfg{merge} && $domid) {
532 $ttl = ($rttl < $ttl ? $rttl : $ttl); # Take the shorter TTL
533 $mergerev->execute(($msg->{isv6} ? 65281 : 65280), $rparent, $ttl, $recid);
534 $dnsdb->_log(rdns_id => $rparent, domain_id => $domid, group_id => $importcfg{group},
535 entry => "[ import ] PTR ".$msg->addr." -> $host merged with matching ".
536 ($msg->{isv6} ? 'AAAA' : 'A')." record");
537 } else {
538 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
539 $recsth->execute(0, $rparent, $host, 12, $msg->addr, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
540 }
541 } else {
542 push @deferred, $rec unless $nodefer;
543 $impok = 0;
544 }
545
546 } elsif ($rec =~ /^\+/) {
547 $filecnt->{A}++;
548
549 my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
550 $host =~ s/^\+//;
551 $host =~ s/\.$//;
552 $host =~ s/^\\052/*/;
553 $ttl = -1 if !defined($ttl) || $ttl eq '';
554 $stamp = '' if !$stamp;
555 $loc = '' if !$loc;
556 $loc = '' if $loc =~ /^:+$/;
557
558 my $stampactive = 'n';
559 my $expires = 'n';
560
561 my $domid = $dnsdb->_hostparent($host);
562 if ($domid) {
563##fixme: really want to pull this DB call inside an if $importcfg{merge},
564# but then we need to duplicate the insert for the case where the matching
565# reverse doesn't exist.
566 $revcheck->execute($host, $ip);
567 my ($revid, $recid, $rttl) = $revcheck->fetchrow_array;
568 if ($importcfg{merge} && $revid) {
569 $ttl = ($rttl < $ttl ? $rttl : $ttl); # Take the shorter TTL
570 $mergefwd->execute(65280, $domid, $ttl, $recid);
571 $dnsdb->_log(rdns_id => $revid, domain_id => $domid, group_id => $importcfg{group},
572 entry => "[ import ] ".($msg->{isv6} ? 'AAAA' : 'A')." record $host -> $ip".
573 " merged with matching PTR record");
574 } else {
575 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
576 $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
577 }
578 } else {
579 push @deferred, $rec unless $nodefer;
580 $impok = 0;
581 }
582
583 } elsif ($rec =~ /^Z/) {
584 $filecnt->{SOA}++;
585
586 my ($zone,$master,$contact,$serial,$refresh,$retry,$expire,$minttl,$ttl,$stamp,$loc) = split /:/, $rec, 11;
587 $zone =~ s/^Z//;
588 $zone =~ s/\.$//;
589 $master =~ s/\.$//;
590 $contact =~ s/\.$//;
591 $ttl = -1 if $ttl eq '';
592 $stamp = '' if !$stamp;
593 $loc = '' if !$loc;
594 $loc = '' if $loc =~ /^:+$/;
595# Default to UNIX epoch for zones with no existing serial value
596 $serial = scalar(time) if !$serial;
597
598 my $stampactive = 'n';
599 my $expires = 'n';
600
601##fixme er... what do we do with an SOA with a timestamp? O_o
602# fail for now, since there's no clean way I can see to handle this (yet)
603# maybe (ab)use the -l flag to import as-is?
604 if ($stamp) {
605 push @deferred, $rec unless $nodefer;
606 return 0;
607 }
608
609##fixme: need more magic on TTL, so we can decide whether to use the minttl or newttl
610# my $newttl;
611# ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'n');
612# $ttl = $newttl if !$ttl;
613
614 if ($zone =~ /\.arpa$/) {
615 ($code,$msg) = DNSDB::_zone2cidr($zone);
616 $dbh->do("INSERT INTO revzones (revnet,group_id,status,default_location,sertype,zserial) VALUES (?,?,1,?,'U',?)",
617 undef, ($msg, $importcfg{group}, $loc, $serial));
618 my ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
619 my $newttl;
620 ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'y');
621 $ttl = $newttl if !$ttl;
622 $recsth->execute(0, $rdns, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl,
623 $loc, $stamp, $expires, $stampactive);
624 } else {
625 $dbh->do("INSERT INTO domains (domain,group_id,status,default_location,sertype,zserial) VALUES (?,?,1,?,'U',?)",
626 undef, ($zone, $importcfg{group}, $loc, $serial));
627 my ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
628 my $newttl;
629 ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'n');
630 $ttl = $newttl if !$ttl;
631 $recsth->execute($domid, 0, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl,
632 $loc, $stamp, $expires, $stampactive);
633 }
634
635 } elsif ($rec =~ /^\@/) {
636 $filecnt->{MX}++;
637
638 my ($zone,$ip,$host,$dist,$ttl,$stamp,$loc) = split /:/, $rec, 7;
639 $zone =~ s/^\@//;
640 $zone =~ s/\.$//;
641 $zone =~ s/^\\052/*/;
642 $host =~ s/\.$//;
643 $host = "$host.mx.$zone" if $host !~ /\./;
644 $ttl = -1 if $ttl eq '';
645 $stamp = '' if !$stamp;
646 $loc = '' if !$loc;
647 $loc = '' if $loc =~ /^:+$/;
648
649 my $stampactive = 'n';
650 my $expires = 'n';
651
652# note we don't check for reverse domains here, because MX records don't make any sense in reverse zones.
653# if this really ever becomes an issue for someone it can be expanded to handle those weirdos
654
655 # allow for subzone MXes, since it's perfectly legitimate to simply stuff it all in a single parent zone
656 my $domid = $dnsdb->_hostparent($zone);
657 if ($domid) {
658 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
659 $recsth->execute($domid, 0, $zone, 15, $host, $dist, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
660 $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive) if $ip;
661 } else {
662 push @deferred, $rec unless $nodefer;
663 $impok = 0;
664 }
665
666 } elsif ($rec =~ /^'/) {
667 $filecnt->{TXT}++;
668
669 my ($fqdn, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 5;
670 $fqdn =~ s/^'//;
671 $fqdn =~ s/^\\052/*/;
672 _deoctal(\$rdata);
673 $ttl = -1 if $ttl eq '';
674 $stamp = '' if !$stamp;
675 $loc = '' if !$loc;
676 $loc = '' if $loc =~ /^:+$/;
677
678 my $stampactive = 'n';
679 my $expires = 'n';
680
681 if ($fqdn =~ /\.arpa$/) {
682 ($code,$msg) = DNSDB::_zone2cidr($fqdn);
683 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
684 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
685 $recsth->execute(0, $rparent, $rdata, 16, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
686 } else {
687 my $domid = $dnsdb->_hostparent($fqdn);
688 if ($domid) {
689 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
690 $recsth->execute($domid, 0, $fqdn, 16, $rdata, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
691 } else {
692 push @deferred, $rec unless $nodefer;
693 $impok = 0;
694 }
695 }
696
697 } elsif ($rec =~ /^\./) {
698 $filecnt->{NSASOA}++;
699
700 my ($fqdn, $ip, $ns, $ttl, $stamp, $loc) = split /:/, $rec, 6;
701 $fqdn =~ s/^\.//;
702 $fqdn =~ s/\.$//;
703 $ns =~ s/\.$//;
704 $ns = "$ns.ns.$fqdn" if $ns !~ /\./;
705 $ttl = -1 if $ttl eq '';
706 $stamp = '' if !$stamp;
707 $loc = '' if !$loc;
708 $loc = '' if $loc =~ /^:+$/;
709
710 my $stampactive = 'n';
711 my $expires = 'n';
712
713##fixme er... what do we do with an SOA with a timestamp? O_o
714# fail for now, since there's no clean way I can see to handle this (yet)
715# maybe (ab)use the -l flag to import as-is?
716 if ($stamp) {
717 push @deferred, $rec unless $nodefer;
718 return 0;
719 }
720
721##fixme: need more magic on TTL, so we can decide whether to use the minttl or newttl
722# my $newttl;
723# ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'n');
724
725 if ($fqdn =~ /\.arpa$/) {
726 ($code,$msg) = DNSDB::_zone2cidr($fqdn);
727 my ($rdns) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ($msg));
728 if (!$rdns) {
729 $errstr = "adding revzone $msg";
730 $dbh->do("INSERT INTO revzones (revnet,group_id,status,default_location) VALUES (?,1,1,?)",
731 undef, ($msg, $loc));
732 ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
733 my $soattl;
734 ($soattl, $stampactive, $expires, $stamp) = calcstamp($stamp, 2560, 0, 'y');
735# this would probably make a lot more sense to do hostmaster.$config{admindomain}
736# otherwise, it's as per the tinydns defaults that work tolerably well on a small scale
737# serial -> modtime of data file, ref -> 16384, ret -> 2048, exp -> 1048576, min -> 2560
738# the SOA also gets the default 2560 TTL, no matter what was set on the . entry.
739 $recsth->execute(0, $rdns, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, $soattl,
740 $loc, $stamp, $expires, $stampactive);
741 }
742 # NS records get the specified TTL from the original . entry
743 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rdns, 'y') if !$stamp;
744 $recsth->execute(0, $rdns, $ns, 2, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
745##fixme: (?) implement full conversion of tinydns . records?
746# -> problem: A record for NS must be added to the appropriate *forward* zone, not the reverse
747#$recsth->execute(0, $rdns, $ns, 1, $ip, 0, 0, 0, $ttl, $stamp, $expires, $stampactive)
748# ... auto-A-record simply does not make sense in reverse zones. Functionally
749# I think it would work, sort of, but it's a nasty mess and anyone hosting reverse
750# zones has names for their nameservers already.
751# Even the auto-nameserver-fqdn comes out... ugly.
752
753 } else {
754 my ($domid) = $dbh->selectrow_array("SELECT domain_id FROM domains WHERE lower(domain) = lower(?)",
755 undef, ($fqdn));
756 if (!$domid) {
757 $errstr = "adding domain $fqdn";
758 $dbh->do("INSERT INTO domains (domain,group_id,status,default_location) VALUES (?,1,1,?)",
759 undef, ($fqdn, $loc));
760 ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
761 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, 2560, 0, 'n');
762 $recsth->execute($domid, 0, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, "2560",
763 $loc, $stamp, $expires, $stampactive);
764 }
765 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n') if !$stamp;
766 $recsth->execute($domid, 0, $fqdn, 2, $ns, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
767 $recsth->execute($domid, 0, $ns, 1, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive) if $ip;
768 }
769
770
771 } elsif ($rec =~ /^\%/) {
772 $filecnt->{VIEWS}++;
773
774 # unfortunate that we don't have a guaranteed way to get a description on these. :/
775 my ($loc,$cnet) = split /:/, $rec, 2;
776 $loc =~ s/^\%//;
777 if (my ($iplist) = $dbh->selectrow_array("SELECT iplist FROM locations WHERE location = ?", undef, ($loc))) {
778 if ($cnet) {
779 $iplist .= ", $cnet";
780 $dbh->do("UPDATE locations SET iplist = ? WHERE location = ?", undef, ($iplist, $loc));
781 } else {
782 # hmm. spit out a warning? if we already have entries for $loc, adding a null
783 # entry will almost certainly Do The Wrong Thing(TM)
784 }
785 } else {
786 $cnet = '' if !$cnet; # de-nullify
787 $dbh->do("INSERT INTO locations (location,iplist,description) VALUES (?,?,?)", undef, ($loc, $cnet, $loc));
788 }
789
790 } elsif ($rec =~ /^:/) {
791 $filecnt->{NCUST}++;
792# Big section. Since tinydns can publish anything you can encode properly, but only provides official
793# recognition and handling for the core common types, this must deal with the leftovers.
794# :fqdn:type:rdata:ttl:time:loc
795
796 my (undef, $fqdn, $type, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 7;
797 $fqdn =~ s/\.$//;
798 $fqdn =~ s/^\\052/*/;
799 $ttl = -1 if $ttl eq '';
800 $stamp = '' if !$stamp;
801 $loc = '' if !$loc;
802 $loc = '' if $loc =~ /^:+$/;
803
804 my $stampactive = 'n';
805 my $expires = 'n';
806
807 if ($type == 33) {
808 # SRV
809 my ($prio, $weight, $port, $target) = (0,0,0,0);
810
811 my @tmp = _byteparse(\$rdata, 2);
812 $prio = $tmp[0] * 256 + $tmp[1];
813 @tmp = _byteparse(\$rdata, 2);
814 $weight = $tmp[0] * 256 + $tmp[1];
815 @tmp = _byteparse(\$rdata, 2);
816 $port = $tmp[0] * 256 + $tmp[1];
817
818 $rdata =~ s/\\\d{3}/./g;
819 ($target) = ($rdata =~ /^\.(.+)\.$/);
820# hmm. the above *should* work, but What If(TM) we have ASCII-range bytes
821# representing the target's fqdn part length(s)? axfr-get doesn't seem to,
822# probably because dec. 33->63 includes most punctuation and all the numbers
823# while ($rdata =~ /(\\\d{3})/) {
824# my $cnt = $1;
825# $rdata =~ s/^$cnt//;
826# $cnt =~ s/^\\/0/;
827# $cnt = oct($cnt);
828# my ($seg) = ($rdata =~ /^(.{$cnt})/);
829# $target .=
830# }
831
832 my $domid = $dnsdb->_hostparent($fqdn);
833 if ($domid) {
834 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
835 $recsth->execute($domid, 0, $fqdn, 33, $target, $prio, $weight, $port, $ttl, $loc, $stamp, $expires, $stampactive) if $domid;
836 } else {
837 push @deferred, $rec unless $nodefer;
838 $impok = 0;
839 }
840
841 } elsif ($type == 28) {
842 # AAAA
843 my @v6;
844
845 for (my $i=0; $i < 8; $i++) {
846 my @tmp = _byteparse(\$rdata, 2);
847 push @v6, sprintf("%0.4x", $tmp[0] * 256 + $tmp[1]);
848 }
849 my $val = NetAddr::IP->new(join(':', @v6));
850
851 my $fparent = $dnsdb->_hostparent($fqdn);
852
853##fixme: really want to pull this DB call inside an if $importcfg{merge},
854# but then we need to duplicate the insert for the case where the matching
855# reverse doesn't exist.
856 $revcheck->execute($fqdn, $val);
857 my ($revid, $recid, $rttl) = $revcheck->fetchrow_array;
858
859 # If we have a revzone and merging is enabled, update the existing
860 # record with a reverse ID, set the type to one of the internal
861 # pseudotypes, and set the TTL to the lower of the two.
862 if ($importcfg{merge} && $revid) {
863 $ttl = ($rttl < $ttl ? $rttl : $ttl); # Take the shorter TTL
864 $mergefwd->execute(65281, $fparent, $ttl, $recid);
865 $dnsdb->_log(rdns_id => $revid, domain_id => $fparent, group_id => $importcfg{group},
866 entry => "[ import ] ".($msg->{isv6} ? 'AAAA' : 'A')." record $fqdn -> $val".
867 " merged with matching PTR record");
868 } else {
869 if ($fparent) {
870 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
871 $recsth->execute($fparent, 0, $fqdn, 28, $val->addr, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
872 } else {
873 push @deferred, $rec unless $nodefer;
874 $impok = 0;
875 }
876 }
877
878 } elsif ($type == 16) {
879 # TXT
880 my $txtstring = _rdata2string($rdata);
881
882 if ($fqdn =~ /\.arpa$/) {
883 ($code,$msg) = DNSDB::_zone2cidr($fqdn);
884 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
885 if ($rparent) {
886 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
887 $recsth->execute(0, $rparent, $txtstring, 16, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
888 } else {
889 push @deferred, $rec unless $nodefer;
890 $impok = 0;
891 }
892 } else {
893 my $domid = $dnsdb->_hostparent($fqdn);
894 if ($domid) {
895 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
896 $recsth->execute($domid, 0, $fqdn, 16, $txtstring, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
897 } else {
898 push @deferred, $rec unless $nodefer;
899 $impok = 0;
900 }
901 }
902
903 } elsif ($type == 17) {
904 # RP
905 my ($email, $txtrec) = split /\\000/, $rdata;
906 $email =~ s/\\\d{3}/./g;
907 $email =~ s/^\.//;
908 $txtrec =~ s/\\\d{3}/./g;
909 $txtrec =~ s/^\.//;
910
911 # these might actually make sense in a reverse zone... sort of.
912 if ($fqdn =~ /\.arpa$/) {
913 ($code,$msg) = DNSDB::_zone2cidr($fqdn);
914 my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
915 if ($rparent) {
916 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
917 $recsth->execute(0, $rparent, "$email $txtrec", 17, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive );
918 } else {
919 push @deferred, $rec unless $nodefer;
920 $impok = 0;
921 }
922 } else {
923 my $domid = $dnsdb->_hostparent($fqdn);
924 if ($domid) {
925 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
926 $recsth->execute($domid, 0, $fqdn, 17, "$email $txtrec", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
927 } else {
928 push @deferred, $rec unless $nodefer;
929 $impok = 0;
930 }
931 }
932
933 } elsif ($type == 44) {
934 # SSHFP
935 my $sshfp = _byteparse(\$rdata, 1);
936 $sshfp .= " "._byteparse(\$rdata, 1);
937 $sshfp .= " "._rdata2hex($rdata);
938
939 # these do not make sense in a reverse zone, since they're logically attached to an A record
940 my $domid = $dnsdb->_hostparent($fqdn);
941 if ($domid) {
942 ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
943 $recsth->execute($domid, 0, $fqdn, 44, $sshfp, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
944 } else {
945 push @deferred, $rec unless $nodefer;
946 $impok = 0;
947 }
948
949 } else {
950 print "unhandled rec $rec\n";
951 $impok = 0;
952 # ... uhhh, dunno
953 }
954
955 } else {
956 $filecnt->{other}++;
957 print " $_\n";
958 }
959
960 return $impok; # just to make sure
961 } # recslurp()
962
963 close FLAT;
964}
Note: See TracBrowser for help on using the repository browser.