source: trunk/vega-import.pl@ 818

#!/usr/bin/perl
# Import script for VegaDNS data
##
# $Id: vega-import.pl 797 2020-11-03 20:38:37Z kdeugau $
# Copyright 2011-2013,2020 Kris Deugau <kdeugau@deepnet.cx>
# 
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version. 
# 
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
# 
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

# Runs in one of two modes:
# - Wipe the contents and import the VegaDNS data with (almost) all entity-ID info as-is
# - Add the contents to the current DB, as a subgroup of the root parent group 1

use strict;
use warnings;

use DBI;
use Data::Dumper;

# push "the directory the script is in" into @INC
use FindBin;
use lib "$FindBin::RealBin/";

use DNSDB;

my $dnsdb = new DNSDB;

my $mode = 'add';

if ($ARGV[0] && $ARGV[0] !~ /^(add|replace)$/) {
  die "Usage:  vega-import.pl [add|replace]\n".
        "       add      Import VegaDNS data as a subgroup of the default root group (default)\n".
        "       replace  Wipe all existing data and import VegaDNS data into the root group\n";
}

$mode = $ARGV[0] if $ARGV[0];

my $newdbh = $dnsdb->{dbh};
$newdbh->{PrintError} = 1;
$newdbh->{PrintWarn} = 1;

my %vegatypes = ('S' => 'SOA', 'N' => 'NS', 'A' => 'A', 'T' => 'TXT',
        'V' => 'SRV', 'P' => 'PTR', 'M' => 'MX', 'C' => 'CNAME',
        '3' => 'AAAA' );

if ($mode eq 'replace') {
  print "WARNING:  Using 'replace' will **DELETE** all existing group,\n".
        "user, domain, and record data in DeepNet DNS Administrator.\n".
        "Once started this cannot be reversed.  Enter 'yes' to proceed: ";
  my $resp = <STDIN>;
  chomp $resp;
  if ($resp ne 'yes') {
    print "Aborting;  no changes made.\n";
    exit;
  } else {
    print "OK, continuing.\n";
    $newdbh->do("DELETE FROM users") or warn "error deleting users: ".$newdbh->errstr."\n";
    $newdbh->do("DELETE FROM log") or warn "error deleting log: ".$newdbh->errstr."\n";
    $newdbh->do("DELETE FROM default_records") or warn "error deleting defrecs: ".$newdbh->errstr."\n";
    $newdbh->do("DELETE FROM records") or warn "error deleting records: ".$newdbh->errstr."\n";
    $newdbh->do("DELETE FROM domains") or warn "error deleting domains: ".$newdbh->errstr."\n";
    $newdbh->do("DELETE FROM groups") or warn "error deleting groups: ".$newdbh->errstr."\n";
    $newdbh->do("DELETE FROM permissions") or warn "error deleting permissions: ".$newdbh->errstr."\n";
    # set seq id on permissions, since we merged the two tables, and the new permission IDs will not match the old
    $newdbh->do("SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence('permissions', 'permission_id'),".
        "1, false)") or warn "couldn't set permission sequence: ".$newdbh->errstr."\n";
    # set seq id on default_records since we're inserting more than we retrieved
    $newdbh->do("SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence('default_records', 'record_id'),".
        "1, false)") or warn "couldn't set defrec sequence: ".$newdbh->errstr."\n";
  }
}

my $vegadbh = DBI->connect('DBI:mysql:database=vegadns;host=localhost', 'vegadns', 'secret', {
        PrintError => 1,
        PrintWarn => 1,
        AutoCommit => 0
  });

my $grpsubs = $vegadbh->prepare("SELECT group_id FROM groups ORDER BY group_id");
my $grpdata = $vegadbh->prepare("SELECT group_id, parent_group_id, name FROM groups WHERE group_id=?");
my $grppget = $vegadbh->prepare("SELECT perm_id,inherit_group_perms,accouedit,accoucreate,accoudelete,".
        "self_edit,group_edit,group_create,group_delete,domain_edit,domain_create,domain_delete,record_edit,".
        "record_create,record_delete FROM group_permissions WHERE group_id=?");
my $userget = $vegadbh->prepare("SELECT user_id, group_id, email, password, first_name, last_name, phone, ".
        "account_type, status FROM accounts WHERE group_id=?");
my $userpget = $vegadbh->prepare("SELECT perm_id,inherit_group_perms,accouedit,accoucreate,accoudelete,".
        "self_edit,group_edit,group_create,group_delete,domain_edit,domain_create,domain_delete,record_edit,".
        "record_create,record_delete FROM user_permissions WHERE user_id=?");
my $domget = $vegadbh->prepare("SELECT domain_id,domain,description,status FROM domains WHERE group_id=?");
my $recget = $vegadbh->prepare("SELECT record_id,host,type,val,distance,weight,port,ttl,description ".
        "FROM records WHERE domain_id=?");
my $defrecget = $vegadbh->prepare("SELECT group_id, host, type, val, distance, weight, port, ttl, ".
        "description FROM default_records WHERE group_id=?");
my $logget = $vegadbh->prepare("SELECT domain_id, user_id, group_id, email, name, entry, time FROM log ".
        "WHERE group_id=?");

my $newgrp = $newdbh->prepare("INSERT INTO groups (group_id,parent_group_id,group_name,permission_id) ".
        "VALUES (?,?,?,?)");
my $newgrppset = $newdbh->prepare("INSERT INTO permissions (self_edit, group_create, ".
        "group_edit, group_delete, user_create, user_edit, user_delete, domain_create, domain_edit, ".
        "domain_delete, record_create, record_edit, record_delete, group_id) ".
        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
my $newpid = $newdbh->prepare("SELECT currval('permissions_permission_id_seq')");
my $newuser = $newdbh->prepare("INSERT INTO users (user_id, group_id, username, password, firstname, ".
        "lastname, phone, type, status, permission_id, inherit_perm) VALUES (?,?,?,?,?,?,?,?,?,?,'f')");
my $newuserpset = $newdbh->prepare("INSERT INTO permissions (admin, self_edit, group_create, ".
        "group_edit, group_delete, user_create, user_edit, user_delete, domain_create, domain_edit, ".
        "domain_delete, record_create, record_edit, record_delete, user_id) ".
        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
my $newdom = $newdbh->prepare("INSERT INTO domains (domain_id, domain, group_id, description, status) VALUES ".
        "(?,?,?,?,?)");
my $newrec = $newdbh->prepare("INSERT INTO records (domain_id,record_id,host,type,val,distance,weight,port,".
        "ttl,description) VALUES (?,?,?,?,?,?,?,?,?,?)");
my $newdefrec = $newdbh->prepare("INSERT INTO default_records (group_id,host,type,val,distance,".
        "weight,port,ttl,description) VALUES (?,?,?,?,?,?,?,?,?)");
my $logput = $newdbh->prepare("INSERT INTO log (domain_id, user_id, group_id, email, name, entry, stamp) ".
        "VALUES (?,?,?,?,?,?,?)");

my $newgrpid = $newdbh->prepare("SELECT currval('groups_group_id_seq')");
my $newpgidup = $newdbh->prepare("UPDATE groups SET permission_id=? WHERE group_id=?");
my $newuserid = $newdbh->prepare("SELECT currval('users_user_id_seq')");
my $newpuidup = $newdbh->prepare("UPDATE users SET permission_id=? WHERE user_id=?");
my $newdomid = $newdbh->prepare("SELECT currval('domains_domain_id_seq')");
my $newrecid = $newdbh->prepare("SELECT currval('records_record_id_seq')");

my $newgrp_add = $newdbh->prepare("INSERT INTO groups (parent_group_id,group_name) VALUES (?,?)");
my $newuser_add = $newdbh->prepare("INSERT INTO users (group_id, username, password, firstname, ".
        "lastname, phone, type, status, inherit_perm) VALUES (?,?,?,?,?,?,?,?,'f')");
my $newdom_add = $newdbh->prepare("INSERT INTO domains (domain, group_id, description, status) VALUES ".
        "(?,?,?,?)");
my $newrec_add = $newdbh->prepare("INSERT INTO records (domain_id,host,type,val,distance,weight,port,".
        "ttl,description) VALUES (?,?,?,?,?,?,?,?,?)");

my $foo = qq(

perm_id | group_id | inherit_group_perms | accouedit | accoucreate | accoudelete | self_edit | group_edit | 
group_create | group_delete | domain_edit | domain_create | domain_delegate | domain_delete | record_edit | 
record_create | record_delete | record_delegate | default_record_edit | default_record_create | 
default_record_delete | rrtype_allow_n | rrtype_allow_a | rrtype_allow_3 | rrtype_allow_6 | rrtype_allow_m | 
rrtype_allow_p | rrtype_allow_t | rrtype_allow_v | rrtype_allow_all

permission_id | admin | self_edit | group_create | group_edit | group_delete | user_create | user_edit | 
user_delete | domain_create | domain_edit | domain_delete | record_create | record_edit | record_delete | 
user_id | group_id

);

my $maxgrp = 0;
my %pcmap;  # used to map parent/child relations

if ($mode eq 'replace') {

  my $stage = 'start';
  $grpsubs->execute;
  my $grp = 0;
  while (($grp) = $grpsubs->fetchrow_array()) {
    $maxgrp = $grp if $grp > $maxgrp;
    dogroup_replace($grp);
  }

  # Fix up sequence numbers to prevent insert problems
  my %idmap = ('groups' => 'group_id', 'users' => 'user_id', 'domains' => 'domain_id',
        'records' => 'record_id');
  for my $table ('groups','users','domains','records') {
    $newdbh->do("SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence('$table','$idmap{$table}'),".
        "(SELECT max($idmap{$table}) FROM $table),true)");
  }

} else {

  $grpsubs->execute;
  my $grp = 0;
  while (($grp) = $grpsubs->fetchrow_array()) {
    $maxgrp = $grp if $grp > $maxgrp;
    dogroup_add($grp);
  }

}

# and done.
exit;

sub dogroup_replace {
  my $grpid = shift;
  $grpdata->execute($grpid) or warn $grpdata->errstr;
  my $oldgrp = $grpdata->fetchrow_hashref() or warn $grpdata->errstr;
  print "group id $grpid, name $oldgrp->{name}\n";

  $grppget->execute($grpid) or warn $grppget->errstr;
  my $oldp;
  unless ($oldp = $grppget->fetchrow_hashref()) {
    foreach my $permtype ('self_edit', 'group_create', 'group_edit', 'group_delete', 'user_create',
        'user_edit', 'user_delete', 'domain_create', 'domain_edit', 'domain_delete', 'record_create',
        'record_edit', 'record_delete') { $oldp->{$permtype} = 't'; }
  }

  # de-nullify
  foreach my $permtype ('self_edit', 'group_create', 'group_edit', 'group_delete', 'user_create',
        'user_edit', 'user_delete', 'domain_create', 'domain_edit', 'domain_delete', 'record_create',
        'record_edit', 'record_delete') {
    $oldp->{$permtype} = 'f' if !defined($oldp->{$permtype});
  }
  $newgrppset->execute($oldp->{self_edit}, $oldp->{group_create}, $oldp->{group_edit}, $oldp->{group_delete},
        $oldp->{user_create}, $oldp->{user_edit}, $oldp->{user_delete},
        $oldp->{domain_create}, $oldp->{domain_edit}, $oldp->{domain_delete},
        $oldp->{record_create}, $oldp->{record_edit}, $oldp->{record_delete},
        $grpid) or warn $newgrppset->errstr;
  $newpid->execute;
  my ($pid) = $newpid->fetchrow_array;
  $newgrp->execute($grpid, $oldgrp->{parent_group_id}, $oldgrp->{name}, $pid);

 ## Users
  print " users: ";
  $userget->execute($grpid);
  while (my $user = $userget->fetchrow_hashref) {
    # fiddle user data
    $user->{account_type} = 'S' if $user->{account_type} eq 'senior_admin';
    $user->{account_type} = 'u' if $user->{account_type} eq 'user';
    $user->{status} = ($user->{status} eq 'active' ? 1 : 0);
    $userpget->execute($user->{user_id}) or warn "failed permission get on ".$user->{user_id}."\n";
    my $oldp = $userpget->fetchrow_hashref;
    # de-nullify
    foreach my $permtype ('admin', 'self_edit', 'group_create', 'group_edit', 'group_delete', 'user_create',
        'user_edit', 'user_delete', 'domain_create', 'domain_edit', 'domain_delete', 'record_create',
        'record_edit', 'record_delete') {
      $oldp->{$permtype} = 'f' if !defined($oldp->{$permtype});
    }
    $newuserpset->execute(($user->{account_type} eq 'S' ? 't' : 'f'), $oldp->{self_edit},
        $oldp->{group_create}, $oldp->{group_edit}, $oldp->{group_delete},
        $oldp->{user_create}, $oldp->{user_edit}, $oldp->{user_delete},
        $oldp->{domain_create}, $oldp->{domain_edit}, $oldp->{domain_delete},
        $oldp->{record_create}, $oldp->{record_edit}, $oldp->{record_delete},
        $user->{user_id}) or warn $newgrppset->errstr;
    $newpid->execute;
    my ($pid) = $newpid->fetchrow_array;
    $newuser->execute($user->{user_id}, $user->{group_id}, $user->{email}, $user->{password},
        $user->{first_name}, $user->{last_name}, $user->{phone}, $user->{account_type}, $user->{status}, $pid) 
        or warn "  new user ".$user->{email}." (".$user->{user_id}.") failed: ".$newuser->errstr."\n";
    print $user->{email}." (".$user->{user_id}."), ";
  }

 ## Domains
  print "\n domains: ";
  $domget->execute($grpid);
  while (my ($id,$dom,$desc,$status) = $domget->fetchrow_array) {
    $status = ($status eq 'active' ? 1 : 0);
    $newdom->execute($id, $dom, $grpid, $desc, $status);
    print "$dom ($id), ";
    $recget->execute($id);
    while (my @rec = $recget->fetchrow_array) {
      $rec[2] = $reverse_typemap{$vegatypes{$rec[2]}};
      $rec[4] = 0 if !$rec[4];
      $rec[5] = 0 if !$rec[5];
      $rec[6] = 0 if !$rec[6];
      $newrec->execute($id,@rec);
    }
  }

 ## Default records
  print "\n default records: ";
  $defrecget->execute(1);       # Vega 1.1.5/1.1.6 do not have default records for all groups;
                                # there is only support for one set of default records coded.
  while (my @rec = $defrecget->fetchrow_array) {
    $rec[0] = $grpid;
    $rec[2] = $reverse_typemap{$vegatypes{$rec[2]}};
    $rec[4] = 0 if !$rec[4];
    $rec[5] = 0 if !$rec[5];
    $rec[6] = 0 if !$rec[6];
    $newdefrec->execute(@rec);
  }

 ## Log entries
  print "\n log entries: ";
  $logget->execute($grpid);
  while (my ($did,$uid,$gid,$email,$name,$entry,$stamp) = $logget->fetchrow_array) {
    $stamp = localtime($stamp).'';
    $logput->execute($did,$uid,$gid,$email,$name,'[Vega] '.$entry,$stamp);
  }

  print "\n done\n";
}

sub dogroup_add {
  my $oldgrpid = shift;

  $grpdata->execute($oldgrpid) or warn $grpdata->errstr;
  my $oldgrp = $grpdata->fetchrow_hashref() or warn $grpdata->errstr;
  print "group id $oldgrpid, name $oldgrp->{name}\n";

  my $newgrpparent = $pcmap{g}{$oldgrp->{parent_group_id}};
  $newgrpparent = 1 if $oldgrpid == 1;

  # do in the same order as dnsdb
  $newgrp_add->execute($newgrpparent, $oldgrp->{name});
  $newgrpid->execute;
  my ($newgid) = $newgrpid->fetchrow_array;
  $pcmap{g}{$oldgrpid} = $newgid;

  $grppget->execute($oldgrpid) or warn $grppget->errstr;
  my $oldp;
  unless ($oldp = $grppget->fetchrow_hashref()) {
    foreach my $permtype ('self_edit', 'group_create', 'group_edit', 'group_delete', 'user_create',
        'user_edit', 'user_delete', 'domain_create', 'domain_edit', 'domain_delete', 'record_create',
        'record_edit', 'record_delete') { $oldp->{$permtype} = 't'; }
  }

  # de-nullify
  foreach my $permtype ('self_edit', 'group_create', 'group_edit', 'group_delete', 'user_create',
        'user_edit', 'user_delete', 'domain_create', 'domain_edit', 'domain_delete', 'record_create',
        'record_edit', 'record_delete') {
    $oldp->{$permtype} = 'f' if !defined($oldp->{$permtype});
  }
  $newgrppset->execute($oldp->{self_edit}, $oldp->{group_create}, $oldp->{group_edit}, $oldp->{group_delete},
        $oldp->{user_create}, $oldp->{user_edit}, $oldp->{user_delete},
        $oldp->{domain_create}, $oldp->{domain_edit}, $oldp->{domain_delete},
        $oldp->{record_create}, $oldp->{record_edit}, $oldp->{record_delete},
        $newgid) or warn $newgrppset->errstr;
  $newpid->execute;
  my ($pid) = $newpid->fetchrow_array;
  $newpgidup->execute($pid,$newgid);

 ## Users
  print " users: ";
  $userget->execute($oldgrpid);
  while (my $user = $userget->fetchrow_hashref) {
    # fiddle user data
    $user->{account_type} = 'S' if $user->{account_type} eq 'senior_admin';
    $user->{account_type} = 'u' if $user->{account_type} eq 'user';
    $user->{status} = ($user->{status} eq 'active' ? 1 : 0);
    $userpget->execute($user->{user_id}) or warn "failed permission get on ".$user->{user_id}."\n";

    $newuser_add->execute($newgid, $user->{email}, $user->{password}, $user->{first_name},
        $user->{last_name}, $user->{phone}, $user->{account_type}, $user->{status}) 
        or warn "  new user ".$user->{email}." (".$user->{user_id}.") failed: ".$newuser->errstr."\n";
    print " user ".$user->{email}." (".$user->{user_id}."), ";
    $newuserid->execute;
    my ($newuid) = $newuserid->fetchrow_array;
    $pcmap{u}{$user->{user_id}} = $newuid;

    my $oldp = $userpget->fetchrow_hashref;
    # de-nullify
    foreach my $permtype ('admin', 'self_edit', 'group_create', 'group_edit', 'group_delete', 'user_create',
        'user_edit', 'user_delete', 'domain_create', 'domain_edit', 'domain_delete', 'record_create',
        'record_edit', 'record_delete') {
      $oldp->{$permtype} = 'f' if !defined($oldp->{$permtype});
    }
    $newuserpset->execute(($user->{account_type} eq 'S' ? 't' : 'f'), $oldp->{self_edit},
        $oldp->{group_create}, $oldp->{group_edit}, $oldp->{group_delete},
        $oldp->{user_create}, $oldp->{user_edit}, $oldp->{user_delete},
        $oldp->{domain_create}, $oldp->{domain_edit}, $oldp->{domain_delete},
        $oldp->{record_create}, $oldp->{record_edit}, $oldp->{record_delete},
        $newuid) or warn $newgrppset->errstr;
    $newpid->execute;
    my ($pid) = $newpid->fetchrow_array;
    $newpuidup->execute($pid,$newuid);
  }

 ## Domains
  print "\n domains: ";
  $domget->execute($oldgrpid);
  while (my ($id,$dom,$desc,$status) = $domget->fetchrow_array) {
    $status = ($status eq 'active' ? 1 : 0);
    $newdom_add->execute($dom, $newgid, $desc, $status);
    print "$dom ($id), ";
    $newdomid->execute;
    my ($newdid) = $newdomid->fetchrow_array;
    $pcmap{d}{$id} = $newdid;
    $recget->execute($id);
    while (my @rec = $recget->fetchrow_array) {
      $rec[0] = $newdid;
      $rec[2] = $reverse_typemap{$vegatypes{$rec[2]}};
      $rec[4] = 0 if !$rec[4];
      $rec[5] = 0 if !$rec[5];
      $rec[6] = 0 if !$rec[6];
      $newrec_add->execute(@rec);
    }
  }

 ## Default records
  print "\n default records: ";
  $defrecget->execute(1);       # Vega 1.1.5/1.1.6 do not have default records for all groups;
                                # there is only support for one set of default records coded.
  while (my @rec = $defrecget->fetchrow_array) {
    $rec[0] = $newgid;
    $rec[2] = $reverse_typemap{$vegatypes{$rec[2]}};
    $rec[4] = 0 if !$rec[4];
    $rec[5] = 0 if !$rec[5];
    $rec[6] = 0 if !$rec[6];
    $newdefrec->execute(@rec);
  }

 ## Log entries
  print "\n log entries: ";
  $logget->execute($oldgrpid);
  while (my ($did,$uid,$gid,$email,$name,$entry,$stamp) = $logget->fetchrow_array) {
    $did = $pcmap{d}{$did};
    $uid = $pcmap{u}{$uid};
    $gid = $pcmap{g}{$gid};
    $stamp = localtime($stamp).'';
    $logput->execute($did,$uid,$gid,$email,$name,'[V] '.$entry,$stamp);
  }

  print "\n done\n";
}