Changeset 371 for trunk/cgi-bin/extras

Timestamp:

11/27/07 12:27:45 (16 years ago)

Author:

Kris Deugau

Message:

/trunk

Merge bugfixes and enhancements from /branches/stable r360 through r369

File:

• trunk/cgi-bin/extras/db2rwhois.pl (modified) (7 diffs)

trunk/cgi-bin/extras/db2rwhois.pl

@@ -10 +10 @@
 # Last update by $Author$
 ###
-# Copyright (C) 2004,2005 - Kris Deugau
+# Copyright (C) 2004-2007 - Kris Deugau
 
 use strict;
@@ -17 +17 @@
 use NetAddr::IP;
 use MyIPDB;
+use File::Path 'rmtree';
 
 $ENV{"PATH"} = "/bin;/usr/bin";
 
 my $rwhoisDataPath = "/etc/rwhoisd";
+
+my @autharea;
+my $authrw;
+# Use the template file to allow us to keep persistent nodes aside from netblock data
+open AUTHTEMPLATE, "<$rwhoisDataPath/rwhoisd.auth_template";
+my $template_persist;
+while (<AUTHTEMPLATE>) {
+  next if /^##/;
+  $template_persist = 1 if /^[a-z]/i;
+  $autharea[0] .= $_;
+}
 
 my ($dbh,$msg) = connectDB_My;
@@ -31 +43 @@
 my %def_custids;
 
+# Get the list of live directories for potential deletion
+opendir RWHOISROOT, $rwhoisDataPath;
+my %rwhoisdirs;
+foreach (readdir RWHOISROOT) {
+  $rwhoisdirs{$_} = 1 if /^net-/;
+}
+closedir RWHOISROOT;
+
+# prefetch alloctype data
+my $sth = $dbh->prepare("select type,def_custid,arin_netname from alloctypes");
+$sth->execute;
+while (my @data = $sth->fetchrow_array) {
+  $netnameprefix{$data[0]} = $data[2];
+}
+
+# Get the list of masters to export
+my $msth = $dbh->prepare("select cidr,ctime,mtime from masterblocks where rwhois='y'");
+$msth->execute;
+
+# Prepare to select subblocks for each master
+# Make sure to remove the private netblocks from this,
+# no use or point in broadcasting our use of them.
+# Also remove the details of our "reserved CORE/WAN" blocks;  they're not critical.
+my $ssth = $dbh->prepare("select cidr,custid,type,city,description,createstamp,modifystamp,swip ".
+        "from allocations where ".
+        "not (cidr <<= '192.168.0.0/16') and ".
+        "not (cidr <<= '172.16.0.0/12') and ".
+        "not (cidr <<= '10.0.0.0/8') and ".
+        "not (type = 'wr') and ".
+        "masklen(cidr) <=30 and ".
+        "cidr <<= ?");
+
+# Customer data, for those rare blocks we really need to delegate.
+my $custsth = $dbh->prepare("select name,street,city,province,country,pocode,phone,tech_handle,special ".
+        "from customers where custid=?");
+
 # Fill in data about our master blocks as allocated from ARIN
 # We open separate files for each of these as appropriate.
-# Note that this ASS-U-MEs that we do not add master IP blocks-
-# there should probably be a separate system for doing that.
-my $sth = $dbh->prepare("select cidr,ctime,mtime from masterblocks;");
-$sth->execute;
+# Changes in master blocks are treated as complete new masters - since we're exporting
+# all data every time, this isn't so terrible as it might seem.
 my $i=0;
-GETMASTERS: while (my @data = $sth->fetchrow_array()) {
-
-# Techically, we only need to exclude 204.138.172.0/24, as we "own" all of the other blocks.
-# However, 205.207.184.0/23 and 206.130.64.0/24 are, um, awkward.
-  if ($data[0] =~ /^(192.168.0.0|172.16.0.0|10.0.0.0|20[456])/) {
-    next GETMASTERS;
-  }
+while (my @data = $msth->fetchrow_array()) {
+
   $masterblocks[$i] = new NetAddr::IP $data[0];
   my ($ctime,undef) = split /\s/, $data[1];
   my ($mtime,undef) = split /\s/, $data[2];
-print "$masterblocks[$i] $ctime $mtime\n";
+
+  print "$masterblocks[$i] $ctime $mtime\n";
 
   my $date;
   chomp ($date = `/bin/date +"%Y-%m-%d"`);
 
-# Whew!  Ugly little varmint.
-  my $masterfilename = "net-".$masterblocks[$i]->addr."-".$masterblocks[$i]->masklen.
-    "/data/network/".$masterblocks[$i]->addr."-".$masterblocks[$i]->masklen.".txt";
-
-# Need check here to create tree for netblock?
+  my $rwnet = "net-".$masterblocks[$i]->addr."-".$masterblocks[$i]->masklen;
+
+  # unflag the directory for deletion.  Whee!  Roundabout!
+  delete $rwhoisdirs{$rwnet};
+
+# Hokay.  Gonna do checks *here* to see if we need to create new master trees
+  my $netdatadir = "$rwhoisDataPath/$rwnet";
+  if (! -e $netdatadir) {
+    print " New master $masterblocks[$i]!\n";
+    print "  Creating directories...\n";
+    mkdir $netdatadir;
+    mkdir "$netdatadir/attribute_defs";
+    mkdir "$netdatadir/data";
+    mkdir "$netdatadir/data/network";
+    mkdir "$netdatadir/data/org";
+    mkdir "$netdatadir/data/referral";
+
+    my $serial;
+    chomp ($serial = `/bin/date '+%Y%m%d'000000000`);
+
+    print "  Creating SOA...\n";
+    open SOAFILE, ">$netdatadir/soa";
+    print SOAFILE qq(Serial-Number: $serial
+Refresh-Interval: 3600
+Increment-Interval: 1800
+Retry-Interval: 1800
+Time-To-Live: 86400
+Primary-Server: rwhois.example.com:4321
+Hostmaster: dns\@example.com
+);
+    close SOAFILE;
+
+    print "  Creating Schema...\n";
+    open SCHEMAFILE, ">$netdatadir/schema";
+    print SCHEMAFILE qq(name: network
+attributedef: $rwnet/attribute_defs/network.tmpl
+dbdir: $rwnet/data/network
+Schema-Version: $serial
+---
+name: organization
+attributedef: $rwnet/attribute_defs/org.tmpl
+dbdir: $rwnet/data/org
+description: Organization object
+Schema-Version: $serial
+---
+name: referral
+attributedef:$rwnet/attribute_defs/referral.tmpl
+dbdir:$rwnet/data/referral
+Schema-Version: $serial
+);
+    close SCHEMAFILE;
+
+    print "  Copying template files...\n";
+    qx { /bin/cp $rwhoisDataPath/skel/attribute_defs/* $netdatadir/attribute_defs/ };
+
+    print "  Creating org data...\n";
+    open ORGDATAFILE, ">$netdatadir/data/org/friendlyisp.txt";
+    print ORGDATAFILE qq(ID: NETBLK-ISP.$masterblocks[$i]
+Auth-Area: $masterblocks[$i]
+Org-Name: Friendly ISP
+Street-Address: 123 4th Street
+City: Anytown
+State: ON
+Postal-Code: H0H 0H0
+Country-Code: CA
+Phone: 000-555-1234
+Created: 20040308
+Updated: 20040308
+);
+    close ORGDATAFILE;
+
+    # Generate auth_area record, and add it to the array.
+    $authrw = 1;        # Flag for rewrite and daemon reload/restart
+
+  } # new master
+
+  # do this for all masters, so that we can use this array to export the data
+  # to rwhoisd.auth_area later if we need to
+  push @autharea, qq(type:master
+name:$masterblocks[$i]
+data-dir: $rwnet/data
+schema-file: $rwnet/schema
+soa-file: $rwnet/soa
+);
+
+  # Recreate the net-nnn.nnn.nnn.nnn-nn.txt data file
+  my $masterfilename = "$rwnet/data/network/".$masterblocks[$i]->addr."-".$masterblocks[$i]->masklen.".txt";
 
   open MASTERFILE,">$rwhoisDataPath/$masterfilename";
@@ -77 +201 @@
         "Updated-By: noc\@example.com\n";
 
-  close MASTERFILE;
-  $i++;
-}
-
-# prefetch alloctype data
-$sth = $dbh->prepare("select type,def_custid,arin_netname from alloctypes");
-$sth->execute;
-while (my @data = $sth->fetchrow_array) {
-  $netnameprefix{$data[0]} = $data[2];
-  $def_custids{$data[0]} = $data[1];
-}
-
-# Now read out the data in the "main" delegation list, and check it
-# with the master blocks.  We need to do this to decide which rWHOIS
-# "net-xxx.xxx.xxx.xxx-mask" tree the data belongs in.
-
-# Make sure to remove the private netblocks from this.
-# No use or point in broadcasting our use of them.
-# Also remove the details of our "reserved CORE/WAN" blocks;  they're not critical.
-$sth = $dbh->prepare("select cidr,custid,type,city,description,createstamp,modifystamp,swip ".
-        "from allocations where ".
-        "not (cidr <<= '192.168.0.0/16') and ".
-        "not (cidr <<= '172.16.0.0/12') and ".
-        "not (cidr <<= '10.0.0.0/8') and ".
-        "not (type = 'wr') and ".
-        "masklen(cidr) <=30");
-#" and (custid='6750400' or custid like '%-RES' or custid like '%-BUS')");
-$sth->execute;
-
-my $custsth = $dbh->prepare("select name,street,city,province,country,pocode,phone,tech_handle,special from customers where custid=?");
-
-$i=0;
-while (my ($cidr, $custid, $type, $city, $desc, $ctime, $mtime, $swip) = $sth->fetchrow_array) {
+  # And now the subblocks
+  $ssth->execute("$masterblocks[$i]");
+  while (my ($cidr, $custid, $type, $city, $desc, $ctime, $mtime, $swip) = $ssth->fetchrow_array) {
 
 # We get master block info from @masterblocks.
@@ -124 +218 @@
  # Updated-By: noc@example.com
 
-  # Get the "full" network number
-  my $net = new NetAddr::IP $cidr;
+    # Get the "full" network number
+    my $net = new NetAddr::IP $cidr;
 
 # Assumptions:  All data in ipdb is public
 # If not, we need another field to indicate "public/private".
 
-  foreach my $master (@masterblocks) {
-    if ($master->contains($net)) {
-
-# Whew!  Ugly little varmint.
-      my $masterfilename = "net-".$master->addr."-".$master->masklen.
-        "/data/network/".$master->addr."-".$master->masklen.".txt";
-
-      open MASTERFILE,">>$rwhoisDataPath/$masterfilename"
-        or print "File open error: '$!' on '$rwhoisDataPath/$masterfilename'\n";
-
 # cidr custid type city description notes maskbits
 
@@ -145 +229 @@
 if ($desc =~ /^\s*$/) { $desc = 'Friendly ISP'; }
 
-      # Fix up datestamps.  We don't *really* need sub-microsecond resolution on our exports...
-      ($ctime) = ($ctime =~ /^(\d+-\d+-\d+)\s+/);
-      ($mtime) = ($mtime =~ /^(\d+-\d+-\d+)\s+/);
+    # Fix up datestamps.  We don't *really* need sub-microsecond resolution on our exports...
+    ($ctime) = ($ctime =~ /^(\d+-\d+-\d+)\s+/);
+    ($mtime) = ($mtime =~ /^(\d+-\d+-\d+)\s+/);
 
 # Notes:
@@ -166 +250 @@
 #  network:Class-Name:network                   [Provided by rWHOIS protocol]
 
-      my $netname = $netnameprefix{$type};
-
-      if ($swip eq 'y' && $def_custids{$type} eq '') {
-        # def_custids{$type} should only be blank on types that are
-        # allocated direct to customers.  i are sMart!!11!!oneone!
-        $custsth->execute($custid);
-        my ($name, $street, $city, $prov, $country, $pocode, $phone, $tech, $special) = $custsth->fetchrow_array;
-        $custsth->finish;
-        if ($special && $special =~ /NetName/ && $special =~ /$cidr/) {
-          ($netname) = ($special =~ /NetName$cidr: ([A-Z0-9_-]+)/);
-        } else {
-          $netname .= "-".$net->network;
-        }
-        print MASTERFILE "---\nID: NETBLK-ISP.$master\n".
-                "Auth-Area: $master\n".
-                "Network-Name: $netname\n".
-                "IP-Network: $net\n".
-                "IP-Network-Block: ".$net->range."\n".
-                "Org-Name: $name\n".
-                "Street-Address: $street\n".
-                "City: $city\n".
-                "StateProv: $prov\n".
-                "Postal-Code: $pocode\n".
-                "Country-Code: $country\n".
-                "Tech-Contact: $tech\n".
-                "Created: $ctime\n".
-                "Updated: $mtime\n".
-                "Updated-By: noc\@example.com\n";
+    my $netname = $netnameprefix{$type};
+
+    if ($swip eq 'n') {
+      print MASTERFILE "---\nID: NETBLK-ISP.$masterblocks[$i]\n".
+        "Auth-Area: $masterblocks[$i]\n".
+        "Network-Name: $netname-".$net->network."\n".
+        "IP-Network: $net\n".
+        "IP-Network-Block: ".$net->range."\n".
+        "Org-Name: Friendly ISP\n".
+        "Street-Address: 123 4th Street\n".
+        "City: Anytown\n".
+        "StateProv: Ontario\n".
+        "Postal-Code: H0H 0H0\n".
+        "Country-Code: CA\n".
+        "Tech-Contact: ISP-ARIN-HANDLE\n".
+        "Created: $ctime\n".
+        "Updated: $mtime\n".
+        "Updated-By: noc\@example.com\n";
+    } else {
+      $custsth->execute($custid);
+      my ($name, $street, $city, $prov, $country, $pocode, $phone, $tech, $special) = $custsth->fetchrow_array;
+      $custsth->finish;
+      if ($special && $special =~ /NetName/ && $special =~ /$cidr/) {
+        ($netname) = ($special =~ /NetName$cidr: ([A-Z0-9_-]+)/);
       } else {
-        print MASTERFILE "---\nID: NETBLK-ISP.$master\n".
-                "Auth-Area: $master\n".
-                "Network-Name: $netname-".$net->network."\n".
-                "IP-Network: $net\n".
-                "IP-Network-Block: ".$net->range."\n".
-                "Org-Name: Friendly ISP\n".
-                "Street-Address: 123 4th Street\n".
-                "City: Anytown\n".
-                "StateProv: Ontario\n".
-                "Postal-Code: H0H 0H0\n".
-                "Country-Code: CA\n".
-                "Tech-Contact: ISP-ARIN-HANDLE\n".
-                "Created: $ctime\n".
-                "Updated: $mtime\n".
-                "Updated-By: noc\@example.com\n";
-      } # decide which contact data to use (cust data only on swip==y && def custid=='')
-    } # net in master
-  } # foreach master
-
-
-
-  #  print "$data[0]\t| $data[1]\t| $data[2]\t| $data[3]\t| $data[4]\t| ".
-  #     "$data[5]\t| $data[6]\t| $data[7]\t| $data[8]\t| $data[9]\n";
-  #  print "$data[0]\t| $data[1]\t| $data[2]\t| $data[3]\t| $data[4]\t| ".
-  #     "$data[5]\t| $data[6]\t| $data[7]\t| $data[8]\n";
+        $netname .= "-".$net->network;
+      }
+      print MASTERFILE "---\nID: NETBLK-ISP.$masterblocks[$i]\n".
+        "Auth-Area: $masterblocks[$i]\n".
+        "Network-Name: $netname\n".
+        "IP-Network: $net\n".
+        "IP-Network-Block: ".$net->range."\n".
+        "Org-Name: $name\n".
+        "Street-Address: $street\n".
+        "City: $city\n".
+        "StateProv: $prov\n".
+        "Postal-Code: $pocode\n".
+        "Country-Code: $country\n".
+        "Tech-Contact: $tech\n".
+        "Created: $ctime\n".
+        "Updated: $mtime\n".
+        "Updated-By: noc\@example.com\n";
+    } # swip
+
+  } # while $ssth->fetchrow_array()
+
+  close MASTERFILE;
+
   $i++;
-} # while fetchrow_array()
-
-
+} # while $msth->fetchrow_array()
+
+# Now we see if there's obsolete netdata directories to be deleted,
+# and therefore an auth-area file to regenerate
+foreach my $netdir (keys %rwhoisdirs) {
+  print "deleting obsolete directory $netdir...\n";
+  rmtree ( "$rwhoisDataPath/$netdir", { verbose => 1, error => \my $errlist } );
+  for my $diag (@$errlist) {
+    my ($file, $message) = each %$diag;
+    if ($file eq '') {
+      print "general error: $message\n";
+    }
+  }
+  $authrw = 1;  # there's probably a more efficient place to put this.  Feh.
+}
+
+# Regenerate rwhoisd.auth_area if needed
+if ($authrw) {
+  print "Regenerating auth_area\n";
+  open RWHOISDAUTH, ">$rwhoisDataPath/rwhoisd.auth_area";
+  print RWHOISDAUTH "# WARNING: This file is autogenerated!  Any static nodes should\n".
+                "# be entered in /etc/rwhoisd/rwhoisd.auth_template\n";
+  if ($template_persist) {
+    print RWHOISDAUTH shift @autharea;
+    print RWHOISDAUTH "---\n";
+  }
+  # feh.  we need to know when we're at the end of the loop, because then
+  # we DON'T want to write the separator...
+  for (;@autharea;) {   # my head hurts.
+    print RWHOISDAUTH shift @autharea;
+    print RWHOISDAUTH "---\n" if @autharea;
+  }
+  close RWHOISDAUTH;
+
+  # restart/reload rwhoisd
+  if (-e "$rwhoisDataPath/rwhoisd.pid") {       # no pidfile, no restart.
+    print "Restarting rwhoisd\n";
+    open PIDFILE, "<$rwhoisDataPath/rwhoisd.pid";
+    my ($rwpid) = (<PIDFILE> =~ /^(\d+)/);
+    close PIDFILE;
+    kill 'HUP', $rwpid;
+  }
+}
+
+# and finally
 $dbh->disconnect;