mirror of
https://git.lapiole.org/dani/ansible-roles.git
synced 2025-07-27 00:05:44 +02:00
Update to 2021-12-01 19:13
This commit is contained in:
Daniel Berteaud
328
roles/pve/files/pve-online
Normal file
328
roles/pve/files/pve-online
Normal file
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Getopt::Long;
|
||||
use Config::Simple;
|
||||
use LWP::UserAgent;
|
||||
use JSON;
|
||||
use PVE::QemuServer;
|
||||
use PVE::LXC::Config;
|
||||
use Array::Diff;
|
||||
use Net::Address::IP::Local;
|
||||
use Net::Route::Table;
|
||||
use NetAddr::IP;
|
||||
use Data::Validate::IP qw(is_ipv4);
|
||||
use Data::Dumper;
|
||||
|
||||
$| = 1;
|
||||
|
||||
my $config = '/etc/pve-online.conf';
|
||||
my ($update_routes, $update_gre, $migrate_ipfo) = undef;
|
||||
|
||||
GetOptions(
|
||||
"config=s" => \$config,
|
||||
"update-routes" => \$update_routes,
|
||||
"update-gre" => \$update_gre,
|
||||
"migrate-ipfo=i" => \$migrate_ipfo
|
||||
);
|
||||
|
||||
# Config can be stored in /etc/pve
|
||||
# Lets wait a bit for it to be available
|
||||
if ($config =~ m|^/etc/pve|){
|
||||
my $t = 0;
|
||||
while ($t < 120){
|
||||
if (!-e $config){
|
||||
print "$config not yet available\n";
|
||||
sleep 2;
|
||||
$t += 2;
|
||||
} else {
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!-e $config){
|
||||
die "$config doesn't exist\n";
|
||||
}
|
||||
|
||||
my $cfg = new Config::Simple;
|
||||
$cfg->read($config);
|
||||
my $conf = $cfg->get_block('general');
|
||||
my $lwp = new LWP::UserAgent;
|
||||
my $online_id = undef;
|
||||
|
||||
if (!$conf){
|
||||
die "No general section found in $config\n";
|
||||
}
|
||||
if (!$conf->{online_api}){
|
||||
die "No online_api defined in $config\n";
|
||||
}
|
||||
|
||||
# Set some defaults
|
||||
$conf->{wan_bridge} ||= 'vmbr1';
|
||||
$conf->{migrate_flush_arp} ||= 'yes';
|
||||
|
||||
if ($update_routes){
|
||||
update_routes();
|
||||
}
|
||||
if ($update_gre){
|
||||
if (defined $conf->{vlan_bridge}){
|
||||
update_gre();
|
||||
} else {
|
||||
print "No VLAN bridge defined. Don't setup GRE tunnels\n";
|
||||
}
|
||||
}
|
||||
if ($migrate_ipfo){
|
||||
my $ipfo = [];
|
||||
# We parse the description field which contains the list of ipfo
|
||||
# attached to this VM
|
||||
foreach my $line ( split("\n", get_guest_conf($migrate_ipfo)->{description} || "") ){
|
||||
if ($line =~ m/^\s*ipfo\d*:\s+(\d+\.\d+\.\d+\.\d+)/){
|
||||
my $candidate = $1;
|
||||
if (is_ipv4($candidate)){
|
||||
push @{$ipfo}, $candidate;
|
||||
print "Found IP $candidate assigned to guest $migrate_ipfo\n";
|
||||
} else {
|
||||
print "Found $candidate assigned to guest $migrate_ipfo which doesn't look like a valid IP\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
$online_id ||= get_online_id();
|
||||
# Now we check if we need to migrate IPFO on the local server
|
||||
my $ipfo_diff = Array::Diff->diff(get_ip_fo($online_id), $ipfo);
|
||||
if (scalar @{$ipfo_diff->added} > 0){
|
||||
print "Update needed. " . join(' ', @{$ipfo_diff->added}) . " should be redirected on server $online_id\n";
|
||||
redirect_ipfo($ipfo_diff->added);
|
||||
update_routes();
|
||||
if ( $ENV{PVE_GUEST_TYPE} ne 'lxc' and $conf->{migrate_flush_arp} =~ m/^1|yes|true|on$/i ){
|
||||
set_guest_nic_down_up($migrate_ipfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#####################################
|
||||
# Sub routines
|
||||
#####################################
|
||||
|
||||
# Query Online's API
|
||||
sub query_online_api{
|
||||
my $uri = shift;
|
||||
$uri = '/api/v1' . $uri if ($uri !~ m|^/api/v1|);
|
||||
my $response = $lwp->get("https://api.online.net$uri",
|
||||
"Authorization" => "Authorization: Bearer $conf->{online_api}",
|
||||
);
|
||||
unless ($response->is_success){
|
||||
die "an error occured while querying the API" .
|
||||
"The error is: " . $response->status_line;
|
||||
}
|
||||
return from_json($response->content);
|
||||
}
|
||||
|
||||
# Update routes for ARP Proxy
|
||||
sub update_routes {
|
||||
$online_id ||= get_online_id();
|
||||
# This is the list of IP which we should have
|
||||
# on the routing table
|
||||
my $routes_online = get_ip_fo($online_id);
|
||||
|
||||
# This is the actual list of IP for which we have routes
|
||||
my $routes_local = get_local_routes();
|
||||
|
||||
# Now, we have to remove routes for those in $routes_local but not in $routes_online
|
||||
# And add routes for those in $routes_online but not in $routes_local
|
||||
my $diff = Array::Diff->diff($routes_online, $routes_local);
|
||||
foreach my $route (@{$diff->added}){
|
||||
next if (grep { $_ eq $route } @{$diff->deleted});
|
||||
print "Removing route for $route\n";
|
||||
system(
|
||||
'/sbin/ip',
|
||||
'route',
|
||||
'del',
|
||||
$route,
|
||||
'dev',
|
||||
$conf->{wan_bridge}
|
||||
);
|
||||
}
|
||||
foreach my $route (@{$diff->deleted}){
|
||||
next if (grep { $_ eq $route } @{$diff->added});
|
||||
print "Adding route for $route\n";
|
||||
system(
|
||||
'/sbin/ip',
|
||||
'route',
|
||||
'add',
|
||||
$route . '/32',
|
||||
'dev',
|
||||
$conf->{wan_bridge}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
# Get the list of routes defined on $conf->{wan_bridge}
|
||||
sub get_local_routes {
|
||||
my $ip = [];
|
||||
my $routes = Net::Route::Table->from_system();
|
||||
foreach my $route (@{$routes->all_routes()}){
|
||||
if ($route->{interface} eq $conf->{wan_bridge} and $route->destination()->masklen() == 32){
|
||||
push @{$ip}, $route->destination()->addr();
|
||||
}
|
||||
}
|
||||
return $ip;
|
||||
}
|
||||
|
||||
# Get the list of IP failover assigned to a server. Taks a server ID as only arg
|
||||
sub get_ip_fo {
|
||||
my $srv_id = shift;
|
||||
return get_srv_info($srv_id)->{network}->{ipfo};
|
||||
}
|
||||
|
||||
# Return server info
|
||||
sub get_srv_info {
|
||||
my $srv_id = shift;
|
||||
return query_online_api('/server/info/' . $srv_id);
|
||||
}
|
||||
|
||||
# Return this server's public IP
|
||||
sub get_public_ip {
|
||||
return Net::Address::IP::Local->public_ipv4;
|
||||
}
|
||||
|
||||
# Return Online's server id
|
||||
sub get_online_id {
|
||||
if (-e '/tmp/online_id'){
|
||||
open my $id_file, '</tmp/online_id';
|
||||
$online_id = <$id_file>;
|
||||
close $id_file;
|
||||
} else {
|
||||
my $ip = get_public_ip();
|
||||
foreach my $srv (@{query_online_api('/server')}){
|
||||
my $info = query_online_api($srv);
|
||||
if ($info->{network}->{ip}[0] eq $ip){
|
||||
$online_id = $info->{id};
|
||||
last;
|
||||
}
|
||||
}
|
||||
open my $id_file, ">/tmp/online_id";
|
||||
print $id_file $online_id;
|
||||
close $id_file;
|
||||
}
|
||||
print "My Online's ID is $online_id\n";
|
||||
return $online_id;
|
||||
}
|
||||
|
||||
sub update_gre {
|
||||
# We have to setup GRE tunnels with all other members
|
||||
# to connect $conf->{vlan_bridge} between every nodes
|
||||
# Something like
|
||||
# ovs-vsctl add-port vmbr1 gre0 -- set interface gre0 type=gre options:remote_ip=''10.29.254.2''
|
||||
# We just have to automate this for every nodes of the cluster
|
||||
print "Getting cluster status...\n";
|
||||
my $members = get_cluster_members();
|
||||
print "Found " . scalar @{$members} . " members\n";
|
||||
my $gre = 0;
|
||||
|
||||
print "Counting GRE ports...\n";
|
||||
my @ports = qx(ovs-vsctl list-ports $conf->{vlan_bridge});
|
||||
my @gre_ports = grep { $_ =~ m/^gre/ } @ports;
|
||||
print "Found " . scalar @gre_ports . " GRE ports\n";
|
||||
|
||||
if (scalar @gre_ports ne scalar @{$members} - 1){
|
||||
print "We need to update GRE tunnels\n";
|
||||
# Remove all greX ports from the VLAN bridge
|
||||
foreach my $port ( @ports ){
|
||||
chomp($port);
|
||||
next unless ($port =~ m/^gre\d+$/);
|
||||
print "Removing port $port from $conf->{vlan_bridge}\n";
|
||||
system(
|
||||
'ovs-vsctl',
|
||||
'del-port',
|
||||
$conf->{vlan_bridge},
|
||||
$port
|
||||
);
|
||||
}
|
||||
# And setup one GRE tunnel per node
|
||||
foreach my $member (@{$members}){
|
||||
# We must skip our own node
|
||||
if (Net::Address::IP::Local->connected_to($member) ne $member){
|
||||
print "Adding GRE interface gre$gre to tunnel with $member\n";
|
||||
system(
|
||||
'ovs-vsctl',
|
||||
'add-port',
|
||||
$conf->{vlan_bridge},
|
||||
'gre' . $gre,
|
||||
'--',
|
||||
'set',
|
||||
'interface',
|
||||
'gre' . $gre,
|
||||
'type=gre',
|
||||
'options:remote_ip=' . $member
|
||||
);
|
||||
$gre++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Get the list of members of this proxmox cluster
|
||||
sub get_cluster_members {
|
||||
my $ip = [];
|
||||
foreach my $line (qx(corosync-cmapctl)){
|
||||
push @{$ip}, $1 if ($line =~ m/ip\((\d+\.\d+\.\d+\.\d+)\)/);
|
||||
}
|
||||
return $ip;
|
||||
}
|
||||
|
||||
sub redirect_ipfo {
|
||||
my $ip = shift;
|
||||
print "Redirecting failover IP " . join(', ', @{$ip}) . "\n";
|
||||
my $response = $lwp->post("https://api.online.net/api/v1/server/failover/edit",
|
||||
Authorization => "Authorization: Bearer $conf->{online_api}",
|
||||
Content => {
|
||||
"source" => join(',', @{$ip}),
|
||||
"destination" => get_public_ip()
|
||||
}
|
||||
);
|
||||
unless ($response->is_success){
|
||||
die "an error occured while querying the API" .
|
||||
"The error is: " . $response->status_line;
|
||||
}
|
||||
}
|
||||
|
||||
# Get a VM configuration
|
||||
sub get_guest_conf {
|
||||
if ($ENV{PVE_GUEST_TYPE} eq 'lxc'){
|
||||
return PVE::LXC::Config->load_config(shift);
|
||||
} else {
|
||||
return PVE::QemuConfig->load_config(shift);
|
||||
}
|
||||
}
|
||||
|
||||
# Unplug and plug back nics connected on the WAN bridge
|
||||
# Needed when moving a failover IP to force flushing the ARP cache
|
||||
# in the guest as the gateway doesn't change, but its MAC does
|
||||
sub set_guest_nic_down_up {
|
||||
my $vmid = shift;
|
||||
my $vm_conf = get_guest_conf($vmid);
|
||||
my $nics = [];
|
||||
my @mod_nic = ();
|
||||
foreach my $key (keys %{$vm_conf}){
|
||||
# Only process netX elements, and which are connected on the WAN bridge
|
||||
next unless ($key =~ m/^net\d+/ && $vm_conf->{$key} =~ m/bridge=$conf->{wan_bridge}/);
|
||||
my $props = $vm_conf->{$key};
|
||||
next if ($props =~ m/link_down=0/);
|
||||
push @mod_nic, $key;
|
||||
if (defined &PVE::QemuServer::Monitor::hmp_cmd){
|
||||
PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $key off");
|
||||
} else {
|
||||
# PVE 5.x doesn't have PVE::QemuServer::Monitor::hmp_cmd
|
||||
PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $key off");
|
||||
}
|
||||
}
|
||||
sleep 1;
|
||||
foreach my $nic (@mod_nic){
|
||||
if (defined &PVE::QemuServer::Monitor::hmp_cmd){
|
||||
PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $nic on");
|
||||
} else {
|
||||
PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $nic on");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user