source: trunk/tiny-import.pl@ 854

#!/usr/bin/perl
# dnsadmin shell-based import tool for tinydns flatfiles
##
# $Id: tiny-import.pl 843 2022-04-29 19:45:16Z kdeugau $
# Copyright 2012-2014,2020-2022 Kris Deugau <kdeugau@deepnet.cx>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

# WARNING:  This is NOT a heavy-duty validator;  it is assumed that the data
# being imported is more or less sane.  Only minor structural validation will
# be done to weed out the most broken records.

use strict;
use warnings;
use POSIX;
use Time::TAI64 qw(:tai);

# Taint-safe (ish) voodoo to push "the directory the script is in" into @INC.
use File::Spec ();
use File::Basename ();
my $path;
BEGIN {
    $path = File::Basename::dirname(File::Spec->rel2abs($0));
    if ($path =~ /(.*)/) {
        $path = $1;
    }
}
use lib $path;

use DNSDB;

my $dnsdb = new DNSDB;

usage() if !@ARGV;

my %importcfg = (
        rw      => 0,
        conv    => 0,
        trial   => 0,
        legacy  => 0,
        merge   => 0,
        group   => 1,
        );
my $gnum = '';
# Handle some command-line arguments
while ($ARGV[0] =~ /^-/) {
  my $arg = shift @ARGV;
  usage() if $arg !~ /^-(?:[rclmt]+|g\d*)$/;
  # -r  rewrite imported files to comment imported records
  # -c  coerce/downconvert A+PTR = records to PTR
  # -l  swallow A+PTR as-is
  # -m  merge PTR and A/AAAA as possible
  # -t  trial mode;  don't commit to DB or actually rewrite flatfile (disables -r)
  # -g  import to specified group (name or ID) instead of group 1
  $arg =~ s/^-//;
# for Reasons (none clear), $arg is undefined yet defined, but only when number characters are involved.  Ebbeh?
no warnings qw(uninitialized);
  if ($arg =~ /^g/) {
    if ($arg eq 'g') {
      $importcfg{group} = shift @ARGV;
    } else {
      $arg =~ s/^g//;
      $importcfg{group} = $arg;
    }
  } else {
    my @tmp = split //, $arg;
    foreach (@tmp) {
      $importcfg{rw} = 1 if $_ eq 'r';
      $importcfg{conv} = 1 if $_ eq 'c';
      $importcfg{legacy} = 1 if $_ eq 'l';
      $importcfg{merge} = 1 if $_ eq 'm';
      $importcfg{trial} = 1 if $_ eq 't';
    }
  }
  use warnings qw(uninitialized);
}
$importcfg{rw} = 0 if $importcfg{trial};

# allow group names
if ($importcfg{group} =~ /^\d+$/) {
  $importcfg{groupname} = $dnsdb->groupName($importcfg{group});
} else {
  $importcfg{groupname} = $importcfg{group};
  $importcfg{group} = $dnsdb->groupID($importcfg{groupname});
}

die usage() if $importcfg{group} !~ /^\d+$/;

sub usage {
  die q(usage:  tiny-import.pl [-rclt] [-gnn] [-g name] datafile1 datafile2 ... datafileN ...
        -r  Rewrite all specified data files with a warning header indicating the
            records are now managed by web, and commenting out all imported records.
            The directory containing any given datafile must be writable.
        -c  Convert any A+PTR (=) record to a bare PTR if the forward domain is
            not present in the database.  Note this does NOT look forward through
            a single file, nor across multiple files handled in the same run.
            Multiple passes may be necessary if SOA and = records are heavily
            intermixed and not clustered together.
        -l  (for "legacy")  Force import of A+PTR records as-is.  Mutually exclusive
            with -c.  -l takes precedence as -c is lossy.
        -m  Merge PTR and A or AAAA records to A+PTR or AAAA+PTR records where possible
        -gnnn or -g nnn or -g name
            Import new zones into this group (group name or ID accepted) instead of
            the root/default group 1
        -t  Trial run mode;  spits out records that would be left unimported.
            Disables -r if set.

        -r and -c may be combined (-rc)

        datafileN is any tinydns record data file.
);
}

my $code;
my $dbh = $dnsdb->{dbh};

# collect some things for logging
($dnsdb->{logusername}, undef, undef, undef, undef, undef, $dnsdb->{logfullname}) = getpwuid($<);
$dnsdb->{logfullname} =~ s/,//g;
$dnsdb->{loguserid} = 0;        # not worth setting up a pseudouser the way the RPC system does
$dnsdb->{logusername} = $dnsdb->{logusername}."/tiny-import.pl";
$dnsdb->{logfullname} = $dnsdb->{logusername} if !$dnsdb->{logfullname};
$dnsdb->{logfullname} = $dnsdb->{logfullname}."/tiny-import.pl";

$dbh->{AutoCommit} = 0;
$dbh->{RaiseError} = 1;

my %cnt;
my @deferred;
my $converted = 0;
my $errstr = '';

foreach my $file (@ARGV) {
  my %filecount;
  my $logentry = "Import records from $file: ";
  eval {
    import(file => $file, cnt => \%filecount);
    if (%filecount) {
      foreach (sort keys %filecount) {
        $logentry .= "$_ $filecount{$_}, ";
        $cnt{$_} += $filecount{$_};
      }
      $logentry =~ s/[\s,]+$//;
      $dnsdb->_log(group_id => $importcfg{group}, entry => $logentry);
    }
    $dbh->rollback if $importcfg{trial};
    $dbh->commit unless $importcfg{trial};
  };
  if ($@) {
    print "Failure trying to import $file: $@\n $errstr\n";
    unlink ".$file.$$" if $importcfg{rw};       # cleanup
    $dbh->rollback;
  }
}

# print summary count of record types encountered
foreach (sort keys %cnt) {
  print " $_    $cnt{$_}\n";
}

exit 0;

sub import {
  our %args = @_;
  my $flatfile = $args{file};
  my $filecnt = $args{cnt};
  my @fpath = split '/', $flatfile;
  $fpath[$#fpath] = ".$fpath[$#fpath]";
  my $rwfile = join('/', @fpath);#.".$$";

  open FLAT, "<$flatfile";

  if ($importcfg{rw}) {
    open RWFLAT, ">$rwfile" or die "Couldn't open tempfile $rwfile for rewriting: $!\n";
    print RWFLAT "# WARNING:  Records in this file have been imported to the web UI.\n#\n";
  }

  our $recsth = $dbh->prepare("INSERT INTO records (domain_id,rdns_id,host,type,val,distance,weight,port,ttl,location,stamp,expires,stampactive) ".
        " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)");

  # for A/AAAA records
  our $revcheck = $dbh->prepare("SELECT rdns_id,record_id,ttl FROM records WHERE host=? AND val=? AND type=12");
  our $mergefwd = $dbh->prepare("UPDATE records SET type=?,domain_id=?,ttl=? WHERE record_id=?");
  # for PTR records
  our $fwdcheck = $dbh->prepare("SELECT domain_id,record_id,ttl FROM records WHERE host=? AND val=? AND (type=1 OR type=28)");
  our $mergerev = $dbh->prepare("UPDATE records SET type=?,rdns_id=?,ttl=? WHERE record_id=?");

  my %deleg;

  my $ok = 0;
  while (<FLAT>) {
    if (/^#/ || /^\s*$/) {
      print RWFLAT "#$_" if $importcfg{rw};
      next;
    }
    chomp;
    s/\s*$//;
    s/:+$//;
    my $recstat = recslurp($_, $filecnt);
    $ok++ if $recstat;
    if ($importcfg{rw}) {
      if ($recstat) {
        print RWFLAT "#$_\n";
      } else {
        print RWFLAT "$_\n";
      }
    }
  }

  # Move the rewritten flatfile in place of the original, so that any
  # external export processing will pick up any remaining records.
  if ($importcfg{rw}) {
    close RWFLAT;
    rename "$rwfile", $flatfile;
  }

  # Show the failed records
  foreach (@deferred) {
    print "failed to import $_\n";
  }

##fixme:  hmm.  can't write the record back to the flatfile in the
# main while above, then come down here and import it anyway, can we?
#   # Try the deferred records again, once.
#  foreach (@deferred) {
#    print "trying $_ again\n";
#    recslurp($_, 1);
#  }

  # .. but we can at least say how many records weren't imported.
  print "$ok OK, ".scalar(@deferred)." deferred, $converted downconverted records in $flatfile\n";
  undef @deferred;
  $converted = 0;

  # Sub for various nonstandard types with lots of pure bytes expressed in octal
  # Takes a tinydns rdata string and count, returns a list of $count bytes as well
  # as trimming those logical bytes off the front of the rdata string.
  sub _byteparse {
    my $src = shift;
    my $count = shift;
    my @ret;
    for (my $i = 0; $i < $count; $i++) {
      if ($$src =~ /^\\/) {
        # we should have an octal bit
        my ($tmp) = ($$src =~ /^(\\\d{3})/);
        $tmp =~ s/\\/0/;
        push @ret, oct($tmp);
        $$src =~ s/^\\\d{3}//;
      } else {
        # we seem to have a byte expressed as an ASCII character
        my ($tmp) = ($$src =~ /^(.)/);
        push @ret, ord($tmp);
        $$src =~ s/^.//;
      }
    }
    return @ret;
  }

  # Convert octal-coded bytes back to something resembling normal characters, general case
  sub _deoctal {
    my $targ = shift;
    while ($$targ =~ /\\(\d{3})/) {
      my $sub = chr(oct($1));
      $$targ =~ s/\\$1/$sub/g;
    }
  }

  sub _rdata2string {
    my $rdata = shift;
    my $tmpout = '';
    while ($rdata) {
      my $bytecount = 0;
      if ($rdata =~ /^\\/) {
        ($bytecount) = ($rdata =~ /^(\\\d{3})/);
        $bytecount =~ s/\\/0/;
        $bytecount = oct($bytecount);
        $rdata =~ s/^\\\d{3}//;
      } else {
        ($bytecount) = ($rdata =~ /^(.)/);
        $bytecount = ord($bytecount);
        $rdata =~ s/^.//;
      }
      my @tmp = _byteparse(\$rdata, $bytecount);
      foreach (@tmp) { $tmpout .= chr($_); }
##fixme:  warn or fail on long (>256?  >512?  >321?) strings
    }
    return $tmpout;
  }

  sub _rdata2hex {
    my $rdata = shift;
    my $tmpout = '';
    while ($rdata) {
      my $byte = '';
      if ($rdata =~ /^\\/) {
        ($byte) = ($rdata =~ /^(\\\d{3})/);
        $byte =~ s/\\/0/;
        $tmpout .= sprintf("%0.2x", oct($byte));
        $rdata =~ s/^\\\d{3}//;
      } else {
        ($byte) = ($rdata =~ /^(.)/);
        $tmpout .= sprintf("%0.2x", ord($byte));
        $rdata =~ s/^.//;
      }
    }
    return $tmpout;
  }

  sub calcstamp {
    my $stampin = shift;
    my $ttl = shift;
    my $pzone = shift;
    my $revrec = shift;

    return ($ttl, 'n', 'n', '1970-01-01 00:00:00 -0') if !$stampin;

##fixme  Yes, this fails for records in 2038 sometime.  No, I'm not going to care for a while.
    $stampin = "\@$stampin";    # Time::TAI64 needs the leading @.  Feh.
    my $u = tai2unix($stampin);
    $stampin = strftime("%Y-%m-%d %H:%M:%S %z", localtime($u));
    my $expires = 'n';
    if ($ttl) {
      # TTL can stay put.
    } else {
      # TTL on import is 0, almost certainly wrong.  Get the parent zone's SOA and use the minttl.
      my $soa = $dnsdb->getSOA('n', $revrec, $pzone);
      $ttl = $soa->{minttl};
      $expires = 'y';
    } 
    return ($ttl, 'y', $expires, $stampin);
  }

  sub recslurp {
    my $rec = shift;
    my $filecnt = shift;
    my $nodefer = shift || 0;
    my $impok = 1;
    my $msg;

    $errstr = $rec;  # this way at least we have some idea what went <splat>

    if ($rec =~ /^=/) {
      $filecnt->{'A+PTR'}++;

##fixme:  do checks like this for all types
      if ($rec !~ /^=(?:\*|\\052)?[a-z0-9\._-]+:[\d\.]+:\d*/i) {
        print "bad A+PTR $rec\n";
        return;
      }
      my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
      $host =~ s/^=//;
      $host =~ s/\.$//;
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;
      my $fparent = $dnsdb->_hostparent($host);
      my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($ip));

      my $stampactive = 'n';
      my $expires = 'n';

      # can't set a timestamp on an orphaned record.  we'll actually fail import of this record a little later.
      if ($fparent || $rparent) {
        if ($fparent) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
        } else {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
        }
      }

      if ($fparent && $rparent) {
        $recsth->execute($fparent, $rparent, $host, 65280, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
      } else {
        if ($importcfg{legacy}) {
          # Just import it already!  Record may still be subject to downconversion on editing.
          $fparent = 0 if !$fparent;
          $rparent = 0 if !$rparent;
          if ($fparent || $rparent) {
            $recsth->execute($fparent, $rparent, $host, 65280, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
          } else {
            # No parents found, cowardly refusing to add a dangling record
            push @deferred, $rec unless $nodefer;
            $impok = 0;
          }
        } elsif ($importcfg{conv}) {
          # downconvert A+PTR if forward zone is not found
          $recsth->execute(0, $rparent, $host, 12, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
          $converted++;
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
          #  print "$tmporig deferred;  can't find both forward and reverse zone parents\n";
        }
      }

    } elsif ($rec =~ /^C/) {
      $filecnt->{CNAME}++;

      my ($host,$targ,$ttl,$stamp,$loc) = split /:/, $rec, 5;
      $host =~ s/^C//;
      $host =~ s/\.$//;
      $host =~ s/^\\052/*/;
      $ttl = -1 if !defined($ttl) || $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

      if ($host =~ /\.arpa$/) {
        ($code,$msg) = DNSDB::_zone2cidr($host);
        my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
        if ($rparent) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
          $recsth->execute(0, $rparent, $targ, 5, $msg->addr, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
          #  print "$tmporig deferred;  can't find parent zone\n";
        }

##fixme:  automagically convert manually maintained sub-/24 delegations
#       my ($subip, $zone) = split /\./, $targ, 2;
#       ($code, $msg) = DNSDB::_zone2cidr($zone);
#       push @{$deleg{"$msg"}{iplist}}, $subip;
#print "$msg $subip\n";

      } else {
        my $fparent = $dnsdb->_hostparent($host);
        if ($fparent) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
          $recsth->execute($fparent, 0, $host, 5, $targ, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
          #  print "$tmporig deferred;  can't find parent zone\n";
        }
      }

    } elsif ($rec =~ /^\&/) {
      $filecnt->{NS}++;

      my ($zone,$ip,$ns,$ttl,$stamp,$loc) = split /:/, $rec, 6;
      $zone =~ s/^\&//;
      $zone =~ s/\.$//;
      $ns =~ s/\.$//;
      $ns = "$ns.ns.$zone" if $ns !~ /\./;
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

      if ($zone =~ /\.arpa$/) {
        ($code,$msg) = DNSDB::_zone2cidr($zone);
        my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >>= ?", undef, ("$msg"));
##fixme, in concert with the CNAME check for same;  automagically
# create "delegate" record instead for subzone NSes:  convert above to use = instead of >>=
#  ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"))
#       if !$rparent;
        if ($rparent) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
          $recsth->execute(0, $rparent, $ns, 2, $msg, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
        }
      } else {
        my $fparent = $dnsdb->_hostparent($zone);
        if ($fparent) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
          $recsth->execute($fparent, 0, $zone, 2, $ns, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
          $recsth->execute($fparent, 0, $ns, 2, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive) if $ip;
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
        }
      }

    } elsif ($rec =~ /^\^/) {
      $filecnt->{PTR}++;

      my ($rip,$host,$ttl,$stamp,$loc) = split /:/, $rec, 5;
      $rip =~ s/^\^//;
      $rip =~ s/\.$//;
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

      my $rparent;
      if (my ($i, $z) = ($rip =~ /^(\d+)\.(\d+-(?:\d+\.){4}in-addr.arpa)$/) ) {
        ($code,$msg) = DNSDB::_zone2cidr($z);
        # Exact matches only, because we're in a sub-/24 delegation
##fixme:  flag the type of delegation (range, subnet-with-dash, subnet-with-slash)
# somewhere so we can recover it on export.  probably best to do that in the revzone data.
        ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ("$msg"));
        $z =~ s/^[\d-]+//;
        ($code,$msg) = DNSDB::_zone2cidr("$i.$z");      # Get the actual IP and normalize
      } else {
        ($code,$msg) = DNSDB::_zone2cidr($rip);
        ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"));
      }

      if ($rparent) {
##fixme:  really want to pull this DB call inside an if $importcfg{merge},
# but then we need to duplicate the insert for the case where the matching
# reverse doesn't exist.
        $host =~ s/\.$//g;   # pure sytactic sugar, we don't store this trailing dot.
        $fwdcheck->execute($host, $msg->addr);
        my ($domid, $recid, $rttl) = $fwdcheck->fetchrow_array;
        if ($importcfg{merge} && $domid) {
          $ttl = ($rttl < $ttl ? $rttl : $ttl);        # Take the shorter TTL
          $mergerev->execute(($msg->{isv6} ? 65281 : 65280), $rparent, $ttl, $recid);
          $dnsdb->_log(rdns_id => $rparent, domain_id => $domid, group_id => $importcfg{group},
            entry => "[ import ] PTR ".$msg->addr." -> $host merged with matching ".
                  ($msg->{isv6} ? 'AAAA' : 'A')." record");
        } else {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
          $recsth->execute(0, $rparent, $host, 12, $msg->addr, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        }
      } else {
        push @deferred, $rec unless $nodefer;
        $impok = 0;
      }

    } elsif ($rec =~ /^\+/) {
      $filecnt->{A}++;

      my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
      $host =~ s/^\+//;
      $host =~ s/\.$//;
      $host =~ s/^\\052/*/;
      $ttl = -1 if !defined($ttl) || $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

      my $domid = $dnsdb->_hostparent($host);
      if ($domid) {
##fixme:  really want to pull this DB call inside an if $importcfg{merge},
# but then we need to duplicate the insert for the case where the matching
# reverse doesn't exist.
        $revcheck->execute($host, $ip);
        my ($revid, $recid, $rttl) = $revcheck->fetchrow_array;
        if ($importcfg{merge} && $revid) {
          $ttl = ($rttl < $ttl ? $rttl : $ttl); # Take the shorter TTL
          $mergefwd->execute(65280, $domid, $ttl, $recid);
          $dnsdb->_log(rdns_id => $revid, domain_id => $domid, group_id => $importcfg{group},
            entry => "[ import ] ".($msg->{isv6} ? 'AAAA' : 'A')." record $host -> $ip".
                  " merged with matching PTR record");
        } else {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
          $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        }
      } else {
        push @deferred, $rec unless $nodefer;
        $impok = 0;
      }

    } elsif ($rec =~ /^Z/) {
      $filecnt->{SOA}++;

      my ($zone,$master,$contact,$serial,$refresh,$retry,$expire,$minttl,$ttl,$stamp,$loc) = split /:/, $rec, 11;
      $zone =~ s/^Z//;
      $zone =~ s/\.$//;
      $master =~ s/\.$//;
      $contact =~ s/\.$//;
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;
# Default to UNIX epoch for zones with no existing serial value
      $serial = scalar(time) if !$serial;

      my $stampactive = 'n';
      my $expires = 'n';

##fixme er... what do we do with an SOA with a timestamp?  O_o
# fail for now, since there's no clean way I can see to handle this (yet)
# maybe (ab)use the -l flag to import as-is?
      if ($stamp) {
        push @deferred, $rec unless $nodefer;
        return 0;
      }

##fixme: need more magic on TTL, so we can decide whether to use the minttl or newttl
#      my $newttl;
#      ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'n');
#      $ttl = $newttl if !$ttl;

      if ($zone =~ /\.arpa$/) {
        ($code,$msg) = DNSDB::_zone2cidr($zone);
        $dbh->do("INSERT INTO revzones (revnet,group_id,status,default_location,sertype,zserial) VALUES (?,?,1,?,'U',?)",
                undef, ($msg, $importcfg{group}, $loc, $serial));
        my ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
        my $newttl;
        ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'y');
        $ttl = $newttl if !$ttl;
        $recsth->execute(0, $rdns, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl,
                $loc, $stamp, $expires, $stampactive);
      } else {
        $dbh->do("INSERT INTO domains (domain,group_id,status,default_location,sertype,zserial) VALUES (?,?,1,?,'U',?)",
                undef, ($zone, $importcfg{group}, $loc, $serial));
        my ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
        my $newttl;
        ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'n');
        $ttl = $newttl if !$ttl;
        $recsth->execute($domid, 0, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl,
                $loc, $stamp, $expires, $stampactive);
      }

    } elsif ($rec =~ /^\@/) {
      $filecnt->{MX}++;

      my ($zone,$ip,$host,$dist,$ttl,$stamp,$loc) = split /:/, $rec, 7;
      $zone =~ s/^\@//;
      $zone =~ s/\.$//;
      $zone =~ s/^\\052/*/;
      $host =~ s/\.$//;
      $host = "$host.mx.$zone" if $host !~ /\./;
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

# note we don't check for reverse domains here, because MX records don't make any sense in reverse zones.
# if this really ever becomes an issue for someone it can be expanded to handle those weirdos

      # allow for subzone MXes, since it's perfectly legitimate to simply stuff it all in a single parent zone
      my $domid = $dnsdb->_hostparent($zone);
      if ($domid) {
        ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
        $recsth->execute($domid, 0, $zone, 15, $host, $dist, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive) if $ip;
      } else {
        push @deferred, $rec unless $nodefer;
        $impok = 0;
      }

    } elsif ($rec =~ /^'/) {
      $filecnt->{TXT}++;

      my ($fqdn, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 5;
      $fqdn =~ s/^'//;
      $fqdn =~ s/^\\052/*/;
      _deoctal(\$rdata);
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

      if ($fqdn =~ /\.arpa$/) {
        ($code,$msg) = DNSDB::_zone2cidr($fqdn);
        my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
        ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
        $recsth->execute(0, $rparent, $rdata, 16, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
      } else {
        my $domid = $dnsdb->_hostparent($fqdn);
        if ($domid) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
          $recsth->execute($domid, 0, $fqdn, 16, $rdata, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
        }
      }

    } elsif ($rec =~ /^\./) {
      $filecnt->{NSASOA}++;

      my ($fqdn, $ip, $ns, $ttl, $stamp, $loc) = split /:/, $rec, 6;
      $fqdn =~ s/^\.//;
      $fqdn =~ s/\.$//;
      $ns =~ s/\.$//;
      $ns = "$ns.ns.$fqdn" if $ns !~ /\./;
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

##fixme er... what do we do with an SOA with a timestamp?  O_o
# fail for now, since there's no clean way I can see to handle this (yet)
# maybe (ab)use the -l flag to import as-is?
      if ($stamp) {
        push @deferred, $rec unless $nodefer;
        return 0;
      }

##fixme: need more magic on TTL, so we can decide whether to use the minttl or newttl
#      my $newttl;
#      ($newttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $minttl, 0, 'n');

      if ($fqdn =~ /\.arpa$/) {
        ($code,$msg) = DNSDB::_zone2cidr($fqdn);
        my ($rdns) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ($msg));
        if (!$rdns) {
          $errstr = "adding revzone $msg";
          $dbh->do("INSERT INTO revzones (revnet,group_id,status,default_location) VALUES (?,1,1,?)",
                undef, ($msg, $loc));
          ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
          my $soattl;
          ($soattl, $stampactive, $expires, $stamp) = calcstamp($stamp, 2560, 0, 'y');
# this would probably make a lot more sense to do hostmaster.$config{admindomain}
# otherwise, it's as per the tinydns defaults that work tolerably well on a small scale
# serial -> modtime of data file, ref -> 16384, ret -> 2048, exp -> 1048576, min -> 2560
# the SOA also gets the default 2560 TTL, no matter what was set on the . entry.
          $recsth->execute(0, $rdns, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, $soattl,
                $loc, $stamp, $expires, $stampactive);
        }
        # NS records get the specified TTL from the original . entry
        ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rdns, 'y') if !$stamp;
        $recsth->execute(0, $rdns, $ns, 2, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
##fixme:  (?)  implement full conversion of tinydns . records?
# -> problem:  A record for NS must be added to the appropriate *forward* zone, not the reverse
#$recsth->execute(0, $rdns, $ns, 1, $ip, 0, 0, 0, $ttl, $stamp, $expires, $stampactive)
# ...  auto-A-record simply does not make sense in reverse zones.  Functionally
# I think it would work, sort of, but it's a nasty mess and anyone hosting reverse
# zones has names for their nameservers already.
# Even the auto-nameserver-fqdn comes out...  ugly.

      } else {
        my ($domid) = $dbh->selectrow_array("SELECT domain_id FROM domains WHERE lower(domain) = lower(?)",
                undef, ($fqdn));
        if (!$domid) {
          $errstr = "adding domain $fqdn";
          $dbh->do("INSERT INTO domains (domain,group_id,status,default_location) VALUES (?,1,1,?)",
                undef, ($fqdn, $loc));
          ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, 2560, 0, 'n');
          $recsth->execute($domid, 0, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, "2560",
                $loc, $stamp, $expires, $stampactive);
        }
        ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n') if !$stamp;
        $recsth->execute($domid, 0, $fqdn, 2, $ns, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        $recsth->execute($domid, 0, $ns, 1, $ip, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive) if $ip;
      }


    } elsif ($rec =~ /^\%/) {
      $filecnt->{VIEWS}++;

      # unfortunate that we don't have a guaranteed way to get a description on these.  :/
      my ($loc,$cnet) = split /:/, $rec, 2;
      $loc =~ s/^\%//;
      if (my ($iplist) = $dbh->selectrow_array("SELECT iplist FROM locations WHERE location = ?", undef, ($loc))) {
        if ($cnet) {
          $iplist .= ", $cnet";
          $dbh->do("UPDATE locations SET iplist = ? WHERE location = ?", undef, ($iplist, $loc));
        } else {
          # hmm.  spit out a warning?  if we already have entries for $loc, adding a null
          # entry will almost certainly Do The Wrong Thing(TM)
        }
      } else {
        $cnet = '' if !$cnet;   # de-nullify
        $dbh->do("INSERT INTO locations (location,iplist,description) VALUES (?,?,?)", undef, ($loc, $cnet, $loc));
      }

    } elsif ($rec =~ /^:/) {
      $filecnt->{NCUST}++;
# Big section.  Since tinydns can publish anything you can encode properly, but only provides official
# recognition and handling for the core common types, this must deal with the leftovers.
# :fqdn:type:rdata:ttl:time:loc

      my (undef, $fqdn, $type, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 7;
      $fqdn =~ s/\.$//;
      $fqdn =~ s/^\\052/*/;
      $ttl = -1 if $ttl eq '';
      $stamp = '' if !$stamp;
      $loc = '' if !$loc;
      $loc = '' if $loc =~ /^:+$/;

      my $stampactive = 'n';
      my $expires = 'n';

      if ($type == 33) {
        # SRV
        my ($prio, $weight, $port, $target) = (0,0,0,0);

        my @tmp = _byteparse(\$rdata, 2);
        $prio = $tmp[0] * 256 + $tmp[1];
        @tmp = _byteparse(\$rdata, 2);
        $weight = $tmp[0] * 256 + $tmp[1];
        @tmp = _byteparse(\$rdata, 2);
        $port = $tmp[0] * 256 + $tmp[1];

        $rdata =~ s/\\\d{3}/./g;
        ($target) = ($rdata =~ /^\.(.+)\.$/);
# hmm.  the above *should* work, but What If(TM) we have ASCII-range bytes
# representing the target's fqdn part length(s)?  axfr-get doesn't seem to,
# probably because dec. 33->63 includes most punctuation and all the numbers
#  while ($rdata =~ /(\\\d{3})/) {
#    my $cnt = $1;
#    $rdata =~ s/^$cnt//;
#    $cnt =~ s/^\\/0/;
#    $cnt = oct($cnt);
#    my ($seg) = ($rdata =~ /^(.{$cnt})/);
#    $target .=
#  }

        my $domid = $dnsdb->_hostparent($fqdn);
        if ($domid) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
          $recsth->execute($domid, 0, $fqdn, 33, $target, $prio, $weight, $port, $ttl, $loc, $stamp, $expires, $stampactive) if $domid;
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
        }

      } elsif ($type == 28) {
        # AAAA
        my @v6;

        for (my $i=0; $i < 8; $i++) {
          my @tmp = _byteparse(\$rdata, 2);
          push @v6, sprintf("%0.4x", $tmp[0] * 256 + $tmp[1]);
        }
        my $val = NetAddr::IP->new(join(':', @v6));

        my $fparent = $dnsdb->_hostparent($fqdn);

##fixme:  really want to pull this DB call inside an if $importcfg{merge},
# but then we need to duplicate the insert for the case where the matching
# reverse doesn't exist.
        $revcheck->execute($fqdn, $val);
        my ($revid, $recid, $rttl) = $revcheck->fetchrow_array;

        # If we have a revzone and merging is enabled, update the existing
        # record with a reverse ID, set the type to one of the internal
        # pseudotypes, and set the TTL to the lower of the two.
        if ($importcfg{merge} && $revid) {
          $ttl = ($rttl < $ttl ? $rttl : $ttl); # Take the shorter TTL
          $mergefwd->execute(65281, $fparent, $ttl, $recid);
          $dnsdb->_log(rdns_id => $revid, domain_id => $fparent, group_id => $importcfg{group},
            entry => "[ import ] ".($msg->{isv6} ? 'AAAA' : 'A')." record $fqdn -> $val".
                  " merged with matching PTR record");
        } else {
          if ($fparent) {
            ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $fparent, 'n');
            $recsth->execute($fparent, 0, $fqdn, 28, $val->addr, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
          } else {
            push @deferred, $rec unless $nodefer;
            $impok = 0;
          }
        }

      } elsif ($type == 16) {
        # TXT
        my $txtstring = _rdata2string($rdata);

        if ($fqdn =~ /\.arpa$/) {
          ($code,$msg) = DNSDB::_zone2cidr($fqdn);
          my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
          if ($rparent) {
            ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
            $recsth->execute(0, $rparent, $txtstring, 16, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
          } else {
            push @deferred, $rec unless $nodefer;
            $impok = 0;
          }
        } else {
          my $domid = $dnsdb->_hostparent($fqdn);
          if ($domid) {
            ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
            $recsth->execute($domid, 0, $fqdn, 16, $txtstring, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
          } else {
            push @deferred, $rec unless $nodefer;
            $impok = 0;
          }
        }

      } elsif ($type == 17) {
        # RP
        my ($email, $txtrec) = split /\\000/, $rdata;
        $email =~ s/\\\d{3}/./g;
        $email =~ s/^\.//;
        $txtrec =~ s/\\\d{3}/./g;
        $txtrec =~ s/^\.//;

        # these might actually make sense in a reverse zone...  sort of.
        if ($fqdn =~ /\.arpa$/) {
          ($code,$msg) = DNSDB::_zone2cidr($fqdn);
          my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
          if ($rparent) {
            ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $rparent, 'y');
            $recsth->execute(0, $rparent, "$email $txtrec", 17, "$msg", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive );
          } else {
            push @deferred, $rec unless $nodefer;
            $impok = 0;
          }
        } else {
          my $domid = $dnsdb->_hostparent($fqdn);
          if ($domid) {
            ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
            $recsth->execute($domid, 0, $fqdn, 17, "$email $txtrec", 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
          } else {
            push @deferred, $rec unless $nodefer;
            $impok = 0;
          }
        }

      } elsif ($type == 44) {
        # SSHFP
        my $sshfp = _byteparse(\$rdata, 1);
        $sshfp .= " "._byteparse(\$rdata, 1);
        $sshfp .= " "._rdata2hex($rdata);

        # these do not make sense in a reverse zone, since they're logically attached to an A record
        my $domid = $dnsdb->_hostparent($fqdn);
        if ($domid) {
          ($ttl, $stampactive, $expires, $stamp) = calcstamp($stamp, $ttl, $domid, 'n');
          $recsth->execute($domid, 0, $fqdn, 44, $sshfp, 0, 0, 0, $ttl, $loc, $stamp, $expires, $stampactive);
        } else {
          push @deferred, $rec unless $nodefer;
          $impok = 0;
        }

      } else {
        print "unhandled rec $rec\n";
        $impok = 0;
        # ... uhhh, dunno
      }

    } else {
      $filecnt->{other}++;
      print " $_\n";
    }

    return $impok;      # just to make sure
  } # recslurp()

  close FLAT;
}