Changeset 545 for branches/stable/tiny-import.pl

Timestamp:

12/10/13 17:47:44 (10 years ago)

Author:

Kris Deugau

Message:

/branches/stable

Merge reverse DNS and location work; 2 of mumble

• from r263 through r416

Numerous conflicts due to hand-copy or partial merges

Location:

branches/stable

Files:

• . (modified) (1 prop)
• tiny-import.pl (copied) (copied from trunk/tiny-import.pl ) (9 diffs)

branches/stable/tiny-import.pl

@@ -19 +19 @@
 ##
 
+# WARNING:  This is NOT a heavy-duty validator;  it is assumed that the data
+# being imported is more or less sane.  Only minor structural validation will
+# be done to weed out the most broken records.
+
 use strict;
 use warnings;
@@ -29 +33 @@
 }
 
+usage() if !@ARGV;
+
+my %importcfg = (
+        rw      => 0,
+        conv    => 0,
+        trial   => 0,
+        );
+# Handle some command-line arguments
+while ($ARGV[0] =~ /^-/) {
+  my $arg = shift @ARGV;
+  usage() if $arg !~ /^-[rct]+$/;
+  # -r  rewrite imported files to comment imported records
+  # -c  coerce/downconvert A+PTR = records to PTR
+  # -t  trial mode;  don't commit to DB or actually rewrite flatfile (disables -r)
+  $arg =~ s/^-//;
+  my @tmp = split //, $arg;
+  foreach (@tmp) {
+    $importcfg{rw} = 1 if $_ eq 'r';
+    $importcfg{conv} = 1 if $_ eq 'c';
+    $importcfg{trial} = 1 if $_ eq 't';
+  }
+}
+$importcfg{rw} = 0 if $importcfg{trial};
+
+sub usage {
+  die q(usage:  tiny-import.pl [-r] [-c] datafile1 datafile2 ... datafileN ...
+        -r  Rewrite all specified data files with a warning header indicating the
+            records are now managed by web, and commenting out all imported records.
+            The directory containing any given datafile must be writable.
+        -c  Convert any A+PTR (=) record to a bare PTR if the forward domain is
+            not present in the database.  Note this does NOT look forward through
+            a single file, nor across multiple files handled in the same run.
+            Multiple passes may be necessary if SOA and = records are heavily
+            intermixed and not clustered together.
+        -t  Trial run mode;  spits out records that would be left unimported.
+            Disables -r if set.
+
+        -r and -c may be combined (-rc)
+
+        datafileN is any tinydns record data file.
+);
+}
+
 my $code;
 my ($dbh,$msg) = connectDB($config{dbname}, $config{dbuser}, $config{dbpass}, $config{dbhost});
@@ -38 +85 @@
 my %cnt;
 my @deferred;
+my $errstr = '';
 
 foreach my $file (@ARGV) {
@@ -43 +91 @@
     import(file => $file);
 #    import(file => $file, nosoa => 1);
-    $dbh->rollback;
-#    $dbh->commit;
+    $dbh->rollback if $importcfg{trial};
+    $dbh->commit unless $importcfg{trial};
   };
   if ($@) {
-    print "bleh: $@\n";
-die "die harder\n";
+    print "Failure trying to import $file: $@\n $errstr\n";
+    unlink ".$file.$$" if $importcfg{rw};       # cleanup
+    $dbh->rollback;
   }
 }
 
-  foreach (keys %cnt) {
-    print " $_  $cnt{$_}\n";
-  }
+# print summary count of record types encountered
+foreach (keys %cnt) {
+  print " $_    $cnt{$_}\n";
+}
 
 exit 0;
@@ -61 +111 @@
   our %args = @_;
   my $flatfile = $args{file};
+  my @fpath = split '/', $flatfile;
+  $fpath[$#fpath] = ".$fpath[$#fpath]";
+  my $rwfile = join('/', @fpath);#.".$$";
+
   open FLAT, "<$flatfile";
 
-  our $recsth = $dbh->prepare("INSERT INTO records (domain_id,rdns_id,host,type,val,distance,weight,port,ttl) ".
-        " VALUES (?,?,?,?,?,?,?,?,?)");
-
+  if ($importcfg{rw}) {
+    open RWFLAT, ">$rwfile" or die "Couldn't open tempfile $rwfile for rewriting: $!\n";
+    print RWFLAT "# WARNING:  Records in this file have been imported to the web UI.\n#\n";
+  }
+
+  our $recsth = $dbh->prepare("INSERT INTO records (domain_id,rdns_id,host,type,val,distance,weight,port,ttl,location) ".
+        " VALUES (?,?,?,?,?,?,?,?,?,?)");
+
+  my %deleg;
+
+  my $ok = 0;
   while (<FLAT>) {
-    next if /^#/;
-    next if /^\s*$/;
+    if (/^#/ || /^\s*$/) {
+      print RWFLAT "#$_" if $importcfg{rw};
+      next;
+    }
     chomp;
-    recslurp($_);
-  }
-
-  # Try the deferred records again, once.
+    s/\s*$//;
+    my $recstat = recslurp($_);
+    $ok++ if $recstat;
+    if ($importcfg{rw}) {
+      if ($recstat) {
+        print RWFLAT "#$_\n";
+      } else {
+        print RWFLAT "$_\n";
+      }
+    }
+  }
+
+  # Move the rewritten flatfile in place of the original, so that any
+  # external export processing will pick up any remaining records.
+  if ($importcfg{rw}) {
+    close RWFLAT;
+    rename "$rwfile", $flatfile;
+  }
+
+  # Show the failed records
   foreach (@deferred) {
-  #  print "trying $_ again\n";
-    recslurp($_, 1);
-  }
+    print "failed to import $_\n";
+  }
+
+##fixme:  hmm.  can't write the record back to the flatfile in the
+# main while above, then come down here and import it anyway, can we?
+#   # Try the deferred records again, once.
+#  foreach (@deferred) {
+#    print "trying $_ again\n";
+#    recslurp($_, 1);
+#  }
+
+  # .. but we can at least say how many records weren't imported.
+  print "$ok OK, ".scalar(@deferred)." deferred records in $flatfile\n";
+  $#deferred = -1;
+
 
   # Sub for various nonstandard types with lots of pure bytes expressed in octal
-  # Takes a tinydns rdata string and count, returns a lis of $count bytes as well
+  # Takes a tinydns rdata string and count, returns a list of $count bytes as well
   # as trimming those logical bytes off the front of the rdata string.
   sub _byteparse {
@@ -103 +195 @@
   }
 
+  # Convert octal-coded bytes back to something resembling normal characters, general case
+  sub _deoctal {
+    my $targ = shift;
+    while ($$targ =~ /\\(\d{3})/) {
+      my $sub = chr(oct($1));
+      $$targ =~ s/\\$1/$sub/g;
+    }
+  }
+
+  sub _rdata2string {
+    my $rdata = shift;
+    my $tmpout = '';
+    while ($rdata) {
+      my $bytecount = 0;
+      if ($rdata =~ /^\\/) {
+        ($bytecount) = ($rdata =~ /^(\\\d{3})/);
+        $bytecount =~ s/\\/0/;
+        $bytecount = oct($bytecount);
+        $rdata =~ s/^\\\d{3}//;
+      } else {
+        ($bytecount) = ($rdata =~ /^(.)/);
+        $bytecount = ord($bytecount);
+        $rdata =~ s/^.//;
+      }
+      my @tmp = _byteparse(\$rdata, $bytecount);
+      foreach (@tmp) { $tmpout .= chr($_); }
+##fixme:  warn or fail on long (>256?  >512?  >321?) strings
+    }
+    return $tmpout;
+  }
+
+  sub _rdata2hex {
+    my $rdata = shift;
+    my $tmpout = '';
+    while ($rdata) {
+      my $byte = '';
+      if ($rdata =~ /^\\/) {
+        ($byte) = ($rdata =~ /^(\\\d{3})/);
+        $byte =~ s/\\/0/;
+        $tmpout .= sprintf("%0.2x", oct($byte));
+        $rdata =~ s/^\\\d{3}//;
+      } else {
+        ($byte) = ($rdata =~ /^(.)/);
+        $tmpout .= sprintf("%0.2x", ord($byte));
+        $rdata =~ s/^.//;
+      }
+    }
+    return $tmpout;
+  }
+
+
   sub recslurp {
     my $rec = shift;
     my $nodefer = shift || 0;
+    my $impok = 1;
+
+    $errstr = $rec;  # this way at least we have some idea what went <splat>
 
     if ($rec =~ /^=/) {
       $cnt{APTR}++;
-if ($rec !~ /^=(?:\*|\\052)?[a-z0-9\._-]+:[\d\.]+:\d*/i) {
-  print "bad A+PTR $rec\n";
-  return;
-#=sud-rr-iGi0-1_sud-gw1-iGi4-2.vianet.ca::10.10.10.13:900::in
-}
-      my ($host,$ip,$ttl,$time,$loc) = split /:/, $rec;
+
+##fixme:  do checks like this for all types
+      if ($rec !~ /^=(?:\*|\\052)?[a-z0-9\._-]+:[\d\.]+:\d*/i) {
+        print "bad A+PTR $rec\n";
+        return;
+      }
+      my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
       $host =~ s/^=//;
       $host =~ s/\.$//;
-      $time = '' if !$time;
-      $loc = '' if !$loc;
-      print "bleh, bad A+PTR!  $rec\n" if $loc =~ /:/;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
       my $fparent = DNSDB::_hostparent($dbh, $host);
       my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($ip));
       if ($fparent && $rparent) {
-        $dbh->do("INSERT INTO records (domain_id,rdns_id,host,type,val,ttl) VALUES (?,?,?,?,?,?)", undef,
-                ($fparent, $rparent, $host, 65280, $ip, $ttl));
+        $recsth->execute($fparent, $rparent, $host, 65280, $ip, 0, 0, 0, $ttl, $loc);
       } else {
         push @deferred, $rec unless $nodefer;
+        $impok = 0;
         #  print "$tmporig deferred;  can't find both forward and reverse zone parents\n";
       }
@@ -132 +280 @@
     } elsif ($rec =~ /^C/) {
       $cnt{CNAME}++;
-      my ($host,$targ,$ttl,$time,$loc) = split /:/, $rec;
+
+      my ($host,$targ,$ttl,$stamp,$loc) = split /:/, $rec, 5;
       $host =~ s/^C//;
       $host =~ s/\.$//;
-      $time = '' if !$time;
-      $loc = '' if !$loc;
-      my $fparent = DNSDB::_hostparent($dbh, $host);
-      if ($fparent) {
-
-      } else {
-        push @deferred, $rec unless $nodefer;
-        #  print "$tmporig deferred;  can't find parent zone\n";
+      $host =~ s/^\\052/*/;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+      if ($host =~ /\.arpa$/) {
+        ($code,$msg) = DNSDB::_zone2cidr($host);
+        my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
+        $recsth->execute(0, $rparent, $targ, 5, $msg->addr, 0, 0, 0, $ttl, $loc);
+
+##fixme:  automagically convert manually maintained sub-/24 delegations
+#       my ($subip, $zone) = split /\./, $targ, 2;
+#       ($code, $msg) = DNSDB::_zone2cidr($zone);
+#       push @{$deleg{"$msg"}{iplist}}, $subip;
+#print "$msg $subip\n";
+
+      } else {
+        my $fparent = DNSDB::_hostparent($dbh, $host);
+        if ($fparent) {
+          $recsth->execute($fparent, 0, $host, 5, $targ, 0, 0, 0, $ttl, $loc);
+        } else {
+          push @deferred, $rec unless $nodefer;
+          $impok = 0;
+          #  print "$tmporig deferred;  can't find parent zone\n";
+        }
       }
 
     } elsif ($rec =~ /^\&/) {
       $cnt{NS}++;
+
+      my ($zone,$ip,$ns,$ttl,$stamp,$loc) = split /:/, $rec, 6;
+      $zone =~ s/^\&//;
+      $zone =~ s/\.$//;
+      $ns =~ s/\.$//;
+      $ns = "$ns.ns.$zone" if $ns !~ /\./;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+      if ($zone =~ /\.arpa$/) {
+        ($code,$msg) = DNSDB::_zone2cidr($zone);
+        my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >>= ?", undef, ("$msg"));
+##fixme, in concert with the CNAME check for same;  automagically
+# create "delegate" record instead for subzone NSes:  convert above to use = instead of >>=
+#  ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"))
+#       if !$rparent;
+        if ($rparent) {
+          $recsth->execute(0, $rparent, $ns, 2, $msg, 0, 0, 0, $ttl, $loc);
+        } else {
+          push @deferred, $rec unless $nodefer;
+          $impok = 0;
+        }
+      } else {
+        my $fparent = DNSDB::_hostparent($dbh, $zone);
+        if ($fparent) {
+          $recsth->execute($fparent, 0, $zone, 2, $ns, 0, 0, 0, $ttl, $loc);
+          $recsth->execute($fparent, 0, $ns, 2, $ip, 0, 0, 0, $ttl, $loc) if $ip;
+        } else {
+          push @deferred, $rec unless $nodefer;
+          $impok = 0;
+        }
+      }
+
     } elsif ($rec =~ /^\^/) {
       $cnt{PTR}++;
+
+      my ($rip,$host,$ttl,$stamp,$loc) = split /:/, $rec, 5;
+      $rip =~ s/^\^//;
+      $rip =~ s/\.$//;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+      my $rparent;
+      if (my ($i, $z) = ($rip =~ /^(\d+)\.(\d+-(?:\d+\.){4}in-addr.arpa)$/) ) {
+        ($code,$msg) = DNSDB::_zone2cidr($z);
+        # Exact matches only, because we're in a sub-/24 delegation
+##fixme:  flag the type of delegation (range, subnet-with-dash, subnet-with-slash)
+# somewhere so we can recover it on export.  probably best to do that in the revzone data.
+        ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ("$msg"));
+        $z =~ s/^[\d-]+//;
+        ($code,$msg) = DNSDB::_zone2cidr("$i.$z");      # Get the actual IP and normalize
+      } else {
+        ($code,$msg) = DNSDB::_zone2cidr($rip);
+        ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ("$msg"));
+      }
+      if ($rparent) {
+        $recsth->execute(0, $rparent, $host, 12, $msg->addr, 0, 0, 0, $ttl, $loc);
+      } else {
+        push @deferred, $rec unless $nodefer;
+        $impok = 0;
+      }
+
     } elsif ($rec =~ /^\+/) {
       $cnt{A}++;
+
+      my ($host,$ip,$ttl,$stamp,$loc) = split /:/, $rec, 5;
+      $host =~ s/^\+//;
+      $host =~ s/\.$//;
+      $host =~ s/^\\052/*/;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+
+      my $domid = DNSDB::_hostparent($dbh, $host);
+      if ($domid) {
+        $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl, $loc);
+      } else {
+        push @deferred, $rec unless $nodefer;
+        $impok = 0;
+      }
+
     } elsif ($rec =~ /^Z/) {
       $cnt{SOA}++;
-#Z128.91.209.in-addr.arpa:ns1.vianet.ca.:dnsadmin.vianet.ca.::1209600:1209600:900:900:900:
-      my ($zone,$master,$contact,$serial,$refresh,$retry,$expire,$minttl,$ttl,$time,$loc) = split /:/, $rec;
+
+      my ($zone,$master,$contact,$serial,$refresh,$retry,$expire,$minttl,$ttl,$stamp,$loc) = split /:/, $rec, 11;
       $zone =~ s/^Z//;
       $zone =~ s/\.$//;
       $master =~ s/\.$//;
       $contact =~ s/\.$//;
-      $time = '' if !$time;
-      $loc = '' if !$loc;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
       if ($zone =~ /\.arpa$/) {
         ($code,$msg) = DNSDB::_zone2cidr($zone);
-        $dbh->do("INSERT INTO revzones (revnet,group_id,status) VALUES (?,1,1)", undef, ($msg));
+        $dbh->do("INSERT INTO revzones (revnet,group_id,status,default_location) VALUES (?,1,1,?)",
+                undef, ($msg, $loc));
         my ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
-        $dbh->do("INSERT INTO records (rdns_id,host,type,val,ttl) VALUES (?,?,6,?,?)", undef,
-                ($rdns, "$contact:$master", "$refresh:$retry:$expire:$minttl", $ttl));
-      } else {
-        $dbh->do("INSERT INTO domains (domain,group_id,status) VALUES (?,1,1)", undef, ($zone));
+        $recsth->execute(0, $rdns, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl, $loc);
+      } else {
+        $dbh->do("INSERT INTO domains (domain,group_id,status,default_location) VALUES (?,1,1,?)",
+                undef, ($zone, $loc));
         my ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
-        $dbh->do("INSERT INTO records (rdns_id,host,type,val,ttl) VALUES (?,?,6,?,?)", undef,
-                ($domid, "$contact:$master", "$refresh:$retry:$expire:$minttl", $ttl));
+        $recsth->execute($domid, 0, "$contact:$master", 6, "$refresh:$retry:$expire:$minttl", 0, 0, 0, $ttl, $loc);
       }
 
     } elsif ($rec =~ /^\@/) {
       $cnt{MX}++;
+
+      my ($zone,$ip,$host,$dist,$ttl,$stamp,$loc) = split /:/, $rec, 7;
+      $zone =~ s/^\@//;
+      $zone =~ s/\.$//;
+      $zone =~ s/^\\052/*/;
+      $host =~ s/\.$//;
+      $host = "$host.mx.$zone" if $host !~ /\./;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+
+# note we don't check for reverse domains here, because MX records don't make any sense in reverse zones.
+# if this really ever becomes an issue for someone it can be expanded to handle those weirdos
+
+      # allow for subzone MXes, since it's perfectly legitimate to simply stuff it all in a single parent zone
+      my $domid = DNSDB::_hostparent($dbh, $zone);
+      if ($domid) {
+        $recsth->execute($domid, 0, $zone, 15, $host, $dist, 0, 0, $ttl, $loc);
+        $recsth->execute($domid, 0, $host, 1, $ip, 0, 0, 0, $ttl, $loc) if $ip;
+      } else {
+        push @deferred, $rec unless $nodefer;
+        $impok = 0;
+      }
+
     } elsif ($rec =~ /^'/) {
       $cnt{TXT}++;
 
-sub _deoctal {
-  my $targ = shift;
-  while ($$targ =~ /\\(\d{3})/) {
-    my $sub = chr(oct($1));
-    $$targ =~ s/\\$1/$sub/g;
-  }
-}
-
-      my ($fqdn, $rdata, $ttl, $time, $loc) = split /:/, $rec;
+      my ($fqdn, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 5;
       $fqdn =~ s/^'//;
+      $fqdn =~ s/^\\052/*/;
       _deoctal(\$rdata);
-
-print "$fqdn TXT '$rdata'\n" if $fqdn =~ /^\*/;
-      my $domid = DNSDB::_hostparent($dbh, $fqdn);
-      if ($domid) {
-        $recsth->execute($domid, 0, $fqdn, 16, $rdata, 0, 0, 0, $ttl);
-      } else {
-        push @deferred, $rec unless $nodefer;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+
+      if ($fqdn =~ /\.arpa$/) {
+        ($code,$msg) = DNSDB::_zone2cidr($fqdn);
+        my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
+        $recsth->execute(0, $rparent, $rdata, 16, "$msg", 0, 0, 0, $ttl, $loc);
+      } else {
+        my $domid = DNSDB::_hostparent($dbh, $fqdn);
+        if ($domid) {
+          $recsth->execute($domid, 0, $fqdn, 16, $rdata, 0, 0, 0, $ttl, $loc);
+        } else {
+          push @deferred, $rec unless $nodefer;
+          $impok = 0;
+        }
       }
 
     } elsif ($rec =~ /^\./) {
       $cnt{NSASOA}++;
+
+      my ($fqdn, $ip, $ns, $ttl, $stamp, $loc) = split /:/, $rec, 6;
+      $fqdn =~ s/^\.//;
+      $fqdn =~ s/\.$//;
+      $ns =~ s/\.$//;
+      $ns = "$ns.ns.$fqdn" if $ns !~ /\./;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+
+      if ($fqdn =~ /\.arpa$/) {
+        ($code,$msg) = DNSDB::_zone2cidr($fqdn);
+        my ($rdns) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet = ?", undef, ($msg));
+        if (!$rdns) {
+          $errstr = "adding revzone $msg";
+          $dbh->do("INSERT INTO revzones (revnet,group_id,status,default_location) VALUES (?,1,1,?)",
+                undef, ($msg, $loc));
+          ($rdns) = $dbh->selectrow_array("SELECT currval('revzones_rdns_id_seq')");
+# this would probably make a lot more sense to do hostmaster.$config{admindomain}
+          $recsth->execute(0, $rdns, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, "2560", $loc);
+        }
+        $recsth->execute(0, $rdns, $ns, 2, "$msg", 0, 0, 0, $ttl, $loc);
+##fixme:  (?)  implement full conversion of tinydns . records?
+# -> problem:  A record for NS must be added to the appropriate *forward* zone, not the reverse
+#$recsth->execute(0, $rdns, $ns, 1, $ip, 0, 0, 0, $ttl)
+# ...  auto-A-record simply does not make sense in reverse zones.  Functionally
+# I think it would work, sort of, but it's a nasty mess and anyone hosting reverse
+# zones has names for their nameservers already.
+# Even the auto-nameserver-fqdn comes out...  ugly.
+
+      } else {
+        my ($domid) = $dbh->selectrow_array("SELECT domain_id FROM domains WHERE lower(domain) = lower(?)",
+                undef, ($fqdn));
+        if (!$domid) {
+          $errstr = "adding domain $fqdn";
+          $dbh->do("INSERT INTO domains (domain,group_id,status,default_location) VALUES (?,1,1,?)",
+                undef, ($fqdn, $loc));
+          ($domid) = $dbh->selectrow_array("SELECT currval('domains_domain_id_seq')");
+          $recsth->execute($domid, 0, "hostmaster.$fqdn:$ns", 6, "16384:2048:1048576:2560", 0, 0, 0, "2560", $loc);
+        }
+        $recsth->execute($domid, 0, $fqdn, 2, $ns, 0, 0, 0, $ttl, $loc);
+        $recsth->execute($domid, 0, $ns, 1, $ip, 0, 0, 0, $ttl, $loc) if $ip;
+      }
+
+
+    } elsif ($rec =~ /^\%/) {
+      $cnt{VIEWS}++;
+
+      # unfortunate that we don't have a guaranteed way to get a description on these.  :/
+      my ($loc,$cnet) = split /:/, $rec, 2;
+      $loc =~ s/^\%//;
+      if (my ($iplist) = $dbh->selectrow_array("SELECT iplist FROM locations WHERE location = ?", undef, ($loc))) {
+        if ($cnet) {
+          $iplist .= ", $cnet";
+          $dbh->do("UPDATE locations SET iplist = ? WHERE location = ?", undef, ($iplist, $loc));
+        } else {
+          # hmm.  spit out a warning?  if we already have entries for $loc, adding a null
+          # entry will almost certainly Do The Wrong Thing(TM)
+        }
+      } else {
+        $cnet = '' if !$cnet;   # de-nullify
+        $dbh->do("INSERT INTO locations (location,iplist,description) VALUES (?,?,?)", undef, ($loc, $cnet, $loc));
+      }
+
     } elsif ($rec =~ /^:/) {
       $cnt{NCUST}++;
@@ -206 +547 @@
 # recognition and handling for the core common types, this must deal with the leftovers.
 # :fqdn:type:rdata:ttl:time:loc
-#:mx2.sys.vianet.ca:44:\001\001\215\272\152\152\123\142\120\071\320\106\160\364\107\372\153\116\036\111\247\135:900::sys
-#:_sipfederationtls._tcp.ncstechnology.com:33:\000\144\000\001\023\305\006sipfed\006online\004lync\003com\000:3600::
-
-my (undef, $fqdn, $type, $rdata, $ttl, $time, $loc) = split /:/, $rec;
-
-if ($type == 33) {
-  # SRV
-  my ($prio, $weight, $port, $target) = (0,0,0,0);
-
-  my @tmp = _byteparse(\$rdata, 2);
-  $prio = $tmp[0] * 256 + $tmp[1];
-  @tmp = _byteparse(\$rdata, 2);
-  $weight = $tmp[0] * 256 + $tmp[1];
-  @tmp = _byteparse(\$rdata, 2);
-  $port = $tmp[0] * 256 + $tmp[1];
-
-  $rdata =~ s/\\\d{3}/./g;
-  ($target) = ($rdata =~ /^\.(.+)\.$/);
+
+      my (undef, $fqdn, $type, $rdata, $ttl, $stamp, $loc) = split /:/, $rec, 7;
+      $fqdn =~ s/\.$//;
+      $fqdn =~ s/^\\052/*/;
+      $ttl = 0 if !$ttl;
+      $stamp = '' if !$stamp;
+      $loc = '' if !$loc;
+      $loc = '' if $loc =~ /^:+$/;
+
+      if ($type == 33) {
+        # SRV
+        my ($prio, $weight, $port, $target) = (0,0,0,0);
+
+        my @tmp = _byteparse(\$rdata, 2);
+        $prio = $tmp[0] * 256 + $tmp[1];
+        @tmp = _byteparse(\$rdata, 2);
+        $weight = $tmp[0] * 256 + $tmp[1];
+        @tmp = _byteparse(\$rdata, 2);
+        $port = $tmp[0] * 256 + $tmp[1];
+
+        $rdata =~ s/\\\d{3}/./g;
+        ($target) = ($rdata =~ /^\.(.+)\.$/);
 # hmm.  the above *should* work, but What If(TM) we have ASCII-range bytes
 # representing the target's fqdn part length(s)?  axfr-get doesn't seem to,
@@ -236 +581 @@
 #  }
 
-  my $domid = DNSDB::_hostparent($dbh, $fqdn);
-  if ($domid) {
-    $recsth->execute($domid, 0, $fqdn, $type, $target, $prio, $weight, $port, $ttl) if $domid;
-  } else {
-    push @deferred, $rec unless $nodefer;
-  }
-
-} elsif ($type == 28) {
-  # AAAA
-
-  my @v6;
-
-  for (my $i=0; $i < 8; $i++) {
-    my @tmp = _byteparse(\$rdata, 2);
-    push @v6, sprintf("%0.4x", $tmp[0] * 256 + $tmp[1]);
-  }
-  my $val = NetAddr::IP->new(join(':', @v6));
-
-  my $domid = DNSDB::_hostparent($dbh, $fqdn);
-  if ($domid) {
-    $recsth->execute($domid, 0, $fqdn, $type, $val->addr, 0, 0, 0, $ttl) if $domid;
-  } else {
-    push @deferred, $rec unless $nodefer;
-  }
-
-} else {
-  # ... uhhh, dunno
-}
+        my $domid = DNSDB::_hostparent($dbh, $fqdn);
+        if ($domid) {
+          $recsth->execute($domid, 0, $fqdn, 33, $target, $prio, $weight, $port, $ttl, $loc) if $domid;
+        } else {
+          push @deferred, $rec unless $nodefer;
+          $impok = 0;
+        }
+
+      } elsif ($type == 28) {
+        # AAAA
+        my @v6;
+
+        for (my $i=0; $i < 8; $i++) {
+          my @tmp = _byteparse(\$rdata, 2);
+          push @v6, sprintf("%0.4x", $tmp[0] * 256 + $tmp[1]);
+        }
+        my $val = NetAddr::IP->new(join(':', @v6));
+
+        my $fparent = DNSDB::_hostparent($dbh, $fqdn);
+        if ($fparent) {
+          $recsth->execute($fparent, 0, $fqdn, 28, $val->addr, 0, 0, 0, $ttl, $loc);
+        } else {
+          push @deferred, $rec unless $nodefer;
+          $impok = 0;
+        }
+
+      } elsif ($type == 16) {
+        # TXT
+        my $txtstring = _rdata2string($rdata);
+
+        if ($fqdn =~ /\.arpa$/) {
+          ($code,$msg) = DNSDB::_zone2cidr($fqdn);
+          my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
+          if ($rparent) {
+            $recsth->execute(0, $rparent, $txtstring, 16, "$msg", 0, 0, 0, $ttl, $loc);
+          } else {
+            push @deferred, $rec unless $nodefer;
+            $impok = 0;
+          }
+        } else {
+          my $domid = DNSDB::_hostparent($dbh, $fqdn);
+          if ($domid) {
+            $recsth->execute($domid, 0, $fqdn, 16, $txtstring, 0, 0, 0, $ttl, $loc);
+          } else {
+            push @deferred, $rec unless $nodefer;
+            $impok = 0;
+          }
+        }
+
+      } elsif ($type == 17) {
+        # RP
+        my ($email, $txtrec) = split /\\000/, $rdata;
+        $email =~ s/\\\d{3}/./g;
+        $email =~ s/^\.//;
+        $txtrec =~ s/\\\d{3}/./g;
+        $txtrec =~ s/^\.//;
+
+        # these might actually make sense in a reverse zone...  sort of.
+        if ($fqdn =~ /\.arpa$/) {
+          ($code,$msg) = DNSDB::_zone2cidr($fqdn);
+          my ($rparent) = $dbh->selectrow_array("SELECT rdns_id FROM revzones WHERE revnet >> ?", undef, ($msg));
+          if ($rparent) {
+            $recsth->execute(0, $rparent, "$email $txtrec", 17, "$msg", 0, 0, 0, $ttl, $loc);
+          } else {
+            push @deferred, $rec unless $nodefer;
+            $impok = 0;
+          }
+        } else {
+          my $domid = DNSDB::_hostparent($dbh, $fqdn);
+          if ($domid) {
+            $recsth->execute($domid, 0, $fqdn, 17, "$email $txtrec", 0, 0, 0, $ttl, $loc);
+          } else {
+            push @deferred, $rec unless $nodefer;
+            $impok = 0;
+          }
+        }
+
+      } elsif ($type == 44) {
+        # SSHFP
+        my $sshfp = _byteparse(\$rdata, 1);
+        $sshfp .= " "._byteparse(\$rdata, 1);
+        $sshfp .= " "._rdata2hex($rdata);
+
+        # these do not make sense in a reverse zone, since they're logically attached to an A record
+        my $domid = DNSDB::_hostparent($dbh, $fqdn);
+        if ($domid) {
+          $recsth->execute($domid, 0, $fqdn, 44, $sshfp, 0, 0, 0, $ttl, $loc);
+        } else {
+          push @deferred, $rec unless $nodefer;
+          $impok = 0;
+        }
+
+      } else {
+        print "unhandled rec $rec\n";
+        $impok = 0;
+        # ... uhhh, dunno
+      }
 
     } else {
       $cnt{other}++;
-  print " $_\n";
+      print " $_\n";
     }
-  }
+
+    return $impok;      # just to make sure
+  } # recslurp()
 
   close FLAT;