Changeset 373


Ignore:
Timestamp:
08/03/12 15:41:46 (12 years ago)
Author:
Kris Deugau
Message:

/trunk

Polish tiny-import.pl for live use

  • Add flatfile-rewriting capability; this lets you import "live" flatfiles without fear of record duplication
  • Add command-line flags to rewrite the flatfile(s), convert A+PTR to PTR, and a trial-run make-no-changes flag to see where problem records might be lurking
  • Clean up remnants of debug-scale error handling and formalize behaviour for problem records
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/tiny-import.pl

    r372 r373  
    3333}
    3434
     35usage() if !@ARGV;
     36
     37my %importcfg = (
     38        rw      => 0,
     39        conv    => 0,
     40        trial   => 0,
     41        );
     42# Handle some command-line arguments
     43while ($ARGV[0] =~ /^-/) {
     44  my $arg = shift @ARGV;
     45  usage() if $arg !~ /^-[rct]+$/;
     46  # -r  rewrite imported files to comment imported records
     47  # -c  coerce/downconvert A+PTR = records to PTR
     48  # -t  trial mode;  don't commit to DB or actually rewrite flatfile (disables -r)
     49  $arg =~ s/^-//;
     50  my @tmp = split //, $arg;
     51  foreach (@tmp) {
     52    $importcfg{rw} = 1 if $_ eq 'r';
     53    $importcfg{conv} = 1 if $_ eq 'c';
     54    $importcfg{trial} = 1 if $_ eq 't';
     55  }
     56}
     57$importcfg{rw} = 0 if $importcfg{trial};
     58
     59sub usage {
     60  die q(usage:  tiny-import.pl [-r] [-c] datafile1 datafile2 ... datafileN ...
     61        -r  Rewrite all specified data files with a warning header indicating the
     62            records are now managed by web, and commenting out all imported records.
     63            The directory containing any given datafile must be writable.
     64        -c  Convert any A+PTR (=) record to a bare PTR if the forward domain is
     65            not present in the database.  Note this does NOT look forward through
     66            a single file, nor across multiple files handled in the same run.
     67            Multiple passes may be necessary if SOA and = records are heavily
     68            intermixed and not clustered together.
     69        -t  Trial run mode;  spits out records that would be left unimported.
     70            Disables -r if set.
     71
     72        -r and -c may be combined (-rc)
     73
     74        datafileN is any tinydns record data file.
     75);
     76}
     77
    3578my $code;
    3679my ($dbh,$msg) = connectDB($config{dbname}, $config{dbuser}, $config{dbpass}, $config{dbhost});
    … …  
    4891    import(file => $file);
    4992#    import(file => $file, nosoa => 1);
    50     $dbh->rollback;
    51 #    $dbh->commit;
     93    $dbh->rollback if $importcfg{trial};
     94    $dbh->commit unless $importcfg{trial};
    5295  };
    5396  if ($@) {
    54     print "bleh: $@\n";
    55 die "die harder: $errstr\n";
     97    print "Failure trying to import $file: $@\n $errstr\n";
     98    unlink ".$file.$$" if $importcfg{rw};       # cleanup
     99    $dbh->rollback;
    56100  }
    57101}
    58102
    59   foreach (keys %cnt) {
    60     print " $_  $cnt{$_}\n";
    61   }
     103# print summary count of record types encountered
     104foreach (keys %cnt) {
     105  print " $_    $cnt{$_}\n";
     106}
    62107
    63108exit 0;
    … …  
    66111  our %args = @_;
    67112  my $flatfile = $args{file};
     113  my @fpath = split '/', $flatfile;
     114  $fpath[$#fpath] = ".$fpath[$#fpath]";
     115  my $rwfile = join('/', @fpath);#.".$$";
     116
    68117  open FLAT, "<$flatfile";
     118
     119  if ($importcfg{rw}) {
     120    open RWFLAT, ">$rwfile" or die "Couldn't open tempfile $rwfile for rewriting: $!\n";
     121    print RWFLAT "# WARNING:  Records in this file have been imported to the web UI.\n#\n";
     122  }
    69123
    70124  our $recsth = $dbh->prepare("INSERT INTO records (domain_id,rdns_id,host,type,val,distance,weight,port,ttl,location) ".
    … …  
    74128
    75129  while (<FLAT>) {
    76     next if /^#/;
    77     next if /^\s*$/;
     130    if (/^#/ || /^\s*$/) {
     131      print RWFLAT "#$_" if $importcfg{rw};
     132      next;
     133    }
    78134    chomp;
    79135    s/\s*$//;
    80     recslurp($_);
    81   }
    82 
    83   # Try the deferred records again, once.
     136    my $recstat = recslurp($_);
     137    if ($importcfg{rw}) {
     138      if ($recstat) {
     139        print RWFLAT "#$_\n";
     140      } else {
     141        print RWFLAT "$_\n";
     142      }
     143    }
     144  }
     145
     146  # Move the rewritten flatfile in place of the original, so that any
     147  # external export processing will pick up any remaining records.
     148  if ($importcfg{rw}) {
     149    close RWFLAT;
     150    rename "$rwfile", $flatfile;
     151  }
     152
     153  # Show the failed records
    84154  foreach (@deferred) {
    85   #  print "trying $_ again\n";
    86     recslurp($_, 1);
    87   }
    88 
    89 print scalar(@deferred)." deferred records in $flatfile\n";
     155    print "failed to import $_\n";
     156  }
     157
     158##fixme:  hmm.  can't write the record back to the flatfile in the
     159# main while above, then come down here and import it anyway, can we?
     160#   # Try the deferred records again, once.
     161#  foreach (@deferred) {
     162#    print "trying $_ again\n";
     163#    recslurp($_, 1);
     164#  }
     165
     166  # .. but we can at least say how many records weren't imported.
     167  print scalar(@deferred)." deferred records in $flatfile\n";
     168  $#deferred = -1;
     169
    90170
    91171  # Sub for various nonstandard types with lots of pure bytes expressed in octal
    … …  
    167247    my $rec = shift;
    168248    my $nodefer = shift || 0;
     249    my $impok = 1;
    169250
    170251    $errstr = $rec;  # this way at least we have some idea what went <splat>
    … …  
    191272      } else {
    192273        push @deferred, $rec unless $nodefer;
     274        $impok = 0;
    193275        #  print "$tmporig deferred;  can't find both forward and reverse zone parents\n";
    194276      }
    … …  
    222304        } else {
    223305          push @deferred, $rec unless $nodefer;
     306          $impok = 0;
    224307          #  print "$tmporig deferred;  can't find parent zone\n";
    225308        }
    … …  
    249332        } else {
    250333          push @deferred, $rec unless $nodefer;
     334          $impok = 0;
    251335        }
    252336      } else {
    … …  
    257341        } else {
    258342          push @deferred, $rec unless $nodefer;
     343          $impok = 0;
    259344        }
    260345      }
    … …  
    287372      } else {
    288373        push @deferred, $rec unless $nodefer;
     374        $impok = 0;
    289375      }
    290376
    … …  
    306392      } else {
    307393        push @deferred, $rec unless $nodefer;
     394        $impok = 0;
    308395      }
    309396
    … …  
    355442      } else {
    356443        push @deferred, $rec unless $nodefer;
     444        $impok = 0;
    357445      }
    358446
    … …  
    379467        } else {
    380468          push @deferred, $rec unless $nodefer;
     469          $impok = 0;
    381470        }
    382471      }
    … …  
    491580        } else {
    492581          push @deferred, $rec unless $nodefer;
     582          $impok = 0;
    493583        }
    494584
    … …  
    508598        } else {
    509599          push @deferred, $rec unless $nodefer;
     600          $impok = 0;
    510601        }
    511602
    … …  
    521612          } else {
    522613            push @deferred, $rec unless $nodefer;
     614            $impok = 0;
    523615          }
    524616        } else {
    … …  
    528620          } else {
    529621            push @deferred, $rec unless $nodefer;
     622            $impok = 0;
    530623          }
    531624        }
    … …  
    547640          } else {
    548641            push @deferred, $rec unless $nodefer;
     642            $impok = 0;
    549643          }
    550644        } else {
    … …  
    554648          } else {
    555649            push @deferred, $rec unless $nodefer;
     650            $impok = 0;
    556651          }
    557652        }
    … …  
    569664        } else {
    570665          push @deferred, $rec unless $nodefer;
    571         }
    572 
    573       } else {
     666          $impok = 0;
     667        }
     668
     669      } else {
     670        print "unhandled rec $rec\n";
     671        $impok = 0;
    574672        # ... uhhh, dunno
    575673      }
    … …  
    579677      print " $_\n";
    580678    }
    581   }
     679
     680    return $impok;      # just to make sure
     681  } # recslurp()
    582682
    583683  close FLAT;
Note: See TracChangeset for help on using the changeset viewer.