mirror of
https://git.lapiole.org/dani/ansible-roles.git
synced 2025-07-27 00:05:44 +02:00
Update to 2021-12-01 19:13
This commit is contained in:
Daniel Berteaud
111
roles/pve/files/online_hook.pl
Executable file
111
roles/pve/files/online_hook.pl
Executable file
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use File::Basename;
|
||||
use JSON;
|
||||
use Logger::Syslog;
|
||||
|
||||
$SIG{CHLD} = 'IGNORE';
|
||||
|
||||
info( "GUEST HOOK: " . join( ' ', @ARGV ) );
|
||||
|
||||
my $vmid = shift;
|
||||
my $phase = shift;
|
||||
|
||||
if ( not -l '/etc/pve/local' ) {
|
||||
error( "Can't find /etc/pve/local link" );
|
||||
die;
|
||||
} elsif ( not defined $vmid or not defined $phase ) {
|
||||
error( "Need to pass both vmid and phase arguments" );
|
||||
die;
|
||||
} elsif ( $vmid !~ m/^\d+$/ ) {
|
||||
error( "vmid must be only numerical" );
|
||||
die;
|
||||
}
|
||||
|
||||
# We must identify the local node
|
||||
my $local_node = basename( readlink( '/etc/pve/local' ) );
|
||||
|
||||
if ( $phase eq 'pre-start' ) {
|
||||
info( "Runing pre-start hook for guest $vmid" );
|
||||
} elsif ( $phase eq 'post-start' ) {
|
||||
info( "Running post-start hook for guest $vmid" );
|
||||
# A VM has just started. Let's check if it's already on the local node
|
||||
# If it's an incoming live migration, it might still be running on another node
|
||||
# All of this must run in the background because hooks are synchronous
|
||||
my $pid = fork();
|
||||
if ( $pid != 0 ) {
|
||||
# main script can stop now, everything will run in a forked process
|
||||
POSIX::_exit 0;
|
||||
} elsif ( defined $pid ) {
|
||||
# All those fh must be closed for the fork to be independant of its parent
|
||||
close STDOUT;
|
||||
close STDERR;
|
||||
close STDIN;
|
||||
POSIX::setsid();
|
||||
|
||||
my $i = 0;
|
||||
my $guest_found = 0;
|
||||
|
||||
# We'll loop for up to 30min, which should be sufficient. If migration takes longer than that,
|
||||
# something is probably wrong
|
||||
LOOP: while ( $i lt 1800 ) {
|
||||
# Here, we query the API for all the VM
|
||||
my $resources = from_json( qx(pvesh get /cluster/resources --type=vm --output-format=json) );
|
||||
# Then we loop through all the VM to find the one we're interested in
|
||||
foreach my $vm ( @{$resources} ){
|
||||
next if ( $vm->{id} !~ m{^(qemu|lxc)/$vmid$} );
|
||||
|
||||
# OK, we found the guest $vmid
|
||||
info("Found guest $vmid, running on node " . $vm->{node});
|
||||
$guest_found = 1;
|
||||
|
||||
# Is the guest running on local node ? If yes, it means migration is finished, and we
|
||||
# can redirect IP failover and routing table
|
||||
if ( $vm->{node} eq $local_node ) {
|
||||
|
||||
# pve-online use this env var to check if we must unplug/replug the WAN NIC
|
||||
$ENV{PVE_GUEST_TYPE} = $1;
|
||||
|
||||
# And here we go !
|
||||
qx(/bin/systemd-cat /usr/local/bin/pve-online --update-routes --update-gre --migrate-ipfo=$vmid);
|
||||
|
||||
# Update routing table of the other online nodes
|
||||
my $nodes = from_json( qx(pvesh get /nodes --output-format=json) );
|
||||
foreach my $node ( @{$nodes} ) {
|
||||
if ( $node->{status} eq 'online' and $node->{node} ne $local_node ) {
|
||||
info("Updating routing table of node $node->{node}");
|
||||
qx(ssh -o ConnectTimeout=3 -l root $node->{node} /usr/local/bin/pve-online --update-routes);
|
||||
}
|
||||
}
|
||||
|
||||
# And we're done, stop looping
|
||||
last LOOP;
|
||||
|
||||
# Guest is not running on the local node = migration is still running
|
||||
# Wait a bit and start again
|
||||
} else {
|
||||
info( "Guest $vmid is still running on node " . $vm->{node} . " not yet on $local_node. Waiting a bit more for migration to finish" );
|
||||
sleep 1;
|
||||
next LOOP;
|
||||
}
|
||||
}
|
||||
|
||||
# We looped through all the guests and couldn't find the one we're looking for, nothing more we can do
|
||||
if ( not $guest_found ) {
|
||||
error( "No such guest with id $vmid" );
|
||||
die;
|
||||
}
|
||||
}
|
||||
}
|
||||
} elsif ( $phase eq 'pre-stop' ) {
|
||||
info( "Running pre-stop hook" );
|
||||
} elsif ( $phase eq 'post-stop' ) {
|
||||
info( "Running post-stop hook" );
|
||||
# Just remove routes if needed
|
||||
qx(/bin/systemd-cat /usr/local/bin/pve-online --update-routes)
|
||||
} else {
|
||||
error( "Unknown hook phase : $phase" );
|
||||
die;
|
||||
}
|
||||
328
roles/pve/files/pve-online
Normal file
328
roles/pve/files/pve-online
Normal file
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Getopt::Long;
|
||||
use Config::Simple;
|
||||
use LWP::UserAgent;
|
||||
use JSON;
|
||||
use PVE::QemuServer;
|
||||
use PVE::LXC::Config;
|
||||
use Array::Diff;
|
||||
use Net::Address::IP::Local;
|
||||
use Net::Route::Table;
|
||||
use NetAddr::IP;
|
||||
use Data::Validate::IP qw(is_ipv4);
|
||||
use Data::Dumper;
|
||||
|
||||
$| = 1;
|
||||
|
||||
my $config = '/etc/pve-online.conf';
|
||||
my ($update_routes, $update_gre, $migrate_ipfo) = undef;
|
||||
|
||||
GetOptions(
|
||||
"config=s" => \$config,
|
||||
"update-routes" => \$update_routes,
|
||||
"update-gre" => \$update_gre,
|
||||
"migrate-ipfo=i" => \$migrate_ipfo
|
||||
);
|
||||
|
||||
# Config can be stored in /etc/pve
|
||||
# Lets wait a bit for it to be available
|
||||
if ($config =~ m|^/etc/pve|){
|
||||
my $t = 0;
|
||||
while ($t < 120){
|
||||
if (!-e $config){
|
||||
print "$config not yet available\n";
|
||||
sleep 2;
|
||||
$t += 2;
|
||||
} else {
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!-e $config){
|
||||
die "$config doesn't exist\n";
|
||||
}
|
||||
|
||||
my $cfg = new Config::Simple;
|
||||
$cfg->read($config);
|
||||
my $conf = $cfg->get_block('general');
|
||||
my $lwp = new LWP::UserAgent;
|
||||
my $online_id = undef;
|
||||
|
||||
if (!$conf){
|
||||
die "No general section found in $config\n";
|
||||
}
|
||||
if (!$conf->{online_api}){
|
||||
die "No online_api defined in $config\n";
|
||||
}
|
||||
|
||||
# Set some defaults
|
||||
$conf->{wan_bridge} ||= 'vmbr1';
|
||||
$conf->{migrate_flush_arp} ||= 'yes';
|
||||
|
||||
if ($update_routes){
|
||||
update_routes();
|
||||
}
|
||||
if ($update_gre){
|
||||
if (defined $conf->{vlan_bridge}){
|
||||
update_gre();
|
||||
} else {
|
||||
print "No VLAN bridge defined. Don't setup GRE tunnels\n";
|
||||
}
|
||||
}
|
||||
if ($migrate_ipfo){
|
||||
my $ipfo = [];
|
||||
# We parse the description field which contains the list of ipfo
|
||||
# attached to this VM
|
||||
foreach my $line ( split("\n", get_guest_conf($migrate_ipfo)->{description} || "") ){
|
||||
if ($line =~ m/^\s*ipfo\d*:\s+(\d+\.\d+\.\d+\.\d+)/){
|
||||
my $candidate = $1;
|
||||
if (is_ipv4($candidate)){
|
||||
push @{$ipfo}, $candidate;
|
||||
print "Found IP $candidate assigned to guest $migrate_ipfo\n";
|
||||
} else {
|
||||
print "Found $candidate assigned to guest $migrate_ipfo which doesn't look like a valid IP\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
$online_id ||= get_online_id();
|
||||
# Now we check if we need to migrate IPFO on the local server
|
||||
my $ipfo_diff = Array::Diff->diff(get_ip_fo($online_id), $ipfo);
|
||||
if (scalar @{$ipfo_diff->added} > 0){
|
||||
print "Update needed. " . join(' ', @{$ipfo_diff->added}) . " should be redirected on server $online_id\n";
|
||||
redirect_ipfo($ipfo_diff->added);
|
||||
update_routes();
|
||||
if ( $ENV{PVE_GUEST_TYPE} ne 'lxc' and $conf->{migrate_flush_arp} =~ m/^1|yes|true|on$/i ){
|
||||
set_guest_nic_down_up($migrate_ipfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#####################################
|
||||
# Sub routines
|
||||
#####################################
|
||||
|
||||
# Query Online's API
|
||||
sub query_online_api{
|
||||
my $uri = shift;
|
||||
$uri = '/api/v1' . $uri if ($uri !~ m|^/api/v1|);
|
||||
my $response = $lwp->get("https://api.online.net$uri",
|
||||
"Authorization" => "Authorization: Bearer $conf->{online_api}",
|
||||
);
|
||||
unless ($response->is_success){
|
||||
die "an error occured while querying the API" .
|
||||
"The error is: " . $response->status_line;
|
||||
}
|
||||
return from_json($response->content);
|
||||
}
|
||||
|
||||
# Update routes for ARP Proxy
|
||||
sub update_routes {
|
||||
$online_id ||= get_online_id();
|
||||
# This is the list of IP which we should have
|
||||
# on the routing table
|
||||
my $routes_online = get_ip_fo($online_id);
|
||||
|
||||
# This is the actual list of IP for which we have routes
|
||||
my $routes_local = get_local_routes();
|
||||
|
||||
# Now, we have to remove routes for those in $routes_local but not in $routes_online
|
||||
# And add routes for those in $routes_online but not in $routes_local
|
||||
my $diff = Array::Diff->diff($routes_online, $routes_local);
|
||||
foreach my $route (@{$diff->added}){
|
||||
next if (grep { $_ eq $route } @{$diff->deleted});
|
||||
print "Removing route for $route\n";
|
||||
system(
|
||||
'/sbin/ip',
|
||||
'route',
|
||||
'del',
|
||||
$route,
|
||||
'dev',
|
||||
$conf->{wan_bridge}
|
||||
);
|
||||
}
|
||||
foreach my $route (@{$diff->deleted}){
|
||||
next if (grep { $_ eq $route } @{$diff->added});
|
||||
print "Adding route for $route\n";
|
||||
system(
|
||||
'/sbin/ip',
|
||||
'route',
|
||||
'add',
|
||||
$route . '/32',
|
||||
'dev',
|
||||
$conf->{wan_bridge}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
# Get the list of routes defined on $conf->{wan_bridge}
|
||||
sub get_local_routes {
|
||||
my $ip = [];
|
||||
my $routes = Net::Route::Table->from_system();
|
||||
foreach my $route (@{$routes->all_routes()}){
|
||||
if ($route->{interface} eq $conf->{wan_bridge} and $route->destination()->masklen() == 32){
|
||||
push @{$ip}, $route->destination()->addr();
|
||||
}
|
||||
}
|
||||
return $ip;
|
||||
}
|
||||
|
||||
# Get the list of IP failover assigned to a server. Taks a server ID as only arg
|
||||
sub get_ip_fo {
|
||||
my $srv_id = shift;
|
||||
return get_srv_info($srv_id)->{network}->{ipfo};
|
||||
}
|
||||
|
||||
# Return server info
|
||||
sub get_srv_info {
|
||||
my $srv_id = shift;
|
||||
return query_online_api('/server/info/' . $srv_id);
|
||||
}
|
||||
|
||||
# Return this server's public IP
|
||||
sub get_public_ip {
|
||||
return Net::Address::IP::Local->public_ipv4;
|
||||
}
|
||||
|
||||
# Return Online's server id
|
||||
sub get_online_id {
|
||||
if (-e '/tmp/online_id'){
|
||||
open my $id_file, '</tmp/online_id';
|
||||
$online_id = <$id_file>;
|
||||
close $id_file;
|
||||
} else {
|
||||
my $ip = get_public_ip();
|
||||
foreach my $srv (@{query_online_api('/server')}){
|
||||
my $info = query_online_api($srv);
|
||||
if ($info->{network}->{ip}[0] eq $ip){
|
||||
$online_id = $info->{id};
|
||||
last;
|
||||
}
|
||||
}
|
||||
open my $id_file, ">/tmp/online_id";
|
||||
print $id_file $online_id;
|
||||
close $id_file;
|
||||
}
|
||||
print "My Online's ID is $online_id\n";
|
||||
return $online_id;
|
||||
}
|
||||
|
||||
sub update_gre {
|
||||
# We have to setup GRE tunnels with all other members
|
||||
# to connect $conf->{vlan_bridge} between every nodes
|
||||
# Something like
|
||||
# ovs-vsctl add-port vmbr1 gre0 -- set interface gre0 type=gre options:remote_ip=''10.29.254.2''
|
||||
# We just have to automate this for every nodes of the cluster
|
||||
print "Getting cluster status...\n";
|
||||
my $members = get_cluster_members();
|
||||
print "Found " . scalar @{$members} . " members\n";
|
||||
my $gre = 0;
|
||||
|
||||
print "Counting GRE ports...\n";
|
||||
my @ports = qx(ovs-vsctl list-ports $conf->{vlan_bridge});
|
||||
my @gre_ports = grep { $_ =~ m/^gre/ } @ports;
|
||||
print "Found " . scalar @gre_ports . " GRE ports\n";
|
||||
|
||||
if (scalar @gre_ports ne scalar @{$members} - 1){
|
||||
print "We need to update GRE tunnels\n";
|
||||
# Remove all greX ports from the VLAN bridge
|
||||
foreach my $port ( @ports ){
|
||||
chomp($port);
|
||||
next unless ($port =~ m/^gre\d+$/);
|
||||
print "Removing port $port from $conf->{vlan_bridge}\n";
|
||||
system(
|
||||
'ovs-vsctl',
|
||||
'del-port',
|
||||
$conf->{vlan_bridge},
|
||||
$port
|
||||
);
|
||||
}
|
||||
# And setup one GRE tunnel per node
|
||||
foreach my $member (@{$members}){
|
||||
# We must skip our own node
|
||||
if (Net::Address::IP::Local->connected_to($member) ne $member){
|
||||
print "Adding GRE interface gre$gre to tunnel with $member\n";
|
||||
system(
|
||||
'ovs-vsctl',
|
||||
'add-port',
|
||||
$conf->{vlan_bridge},
|
||||
'gre' . $gre,
|
||||
'--',
|
||||
'set',
|
||||
'interface',
|
||||
'gre' . $gre,
|
||||
'type=gre',
|
||||
'options:remote_ip=' . $member
|
||||
);
|
||||
$gre++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Get the list of members of this proxmox cluster
|
||||
sub get_cluster_members {
|
||||
my $ip = [];
|
||||
foreach my $line (qx(corosync-cmapctl)){
|
||||
push @{$ip}, $1 if ($line =~ m/ip\((\d+\.\d+\.\d+\.\d+)\)/);
|
||||
}
|
||||
return $ip;
|
||||
}
|
||||
|
||||
sub redirect_ipfo {
|
||||
my $ip = shift;
|
||||
print "Redirecting failover IP " . join(', ', @{$ip}) . "\n";
|
||||
my $response = $lwp->post("https://api.online.net/api/v1/server/failover/edit",
|
||||
Authorization => "Authorization: Bearer $conf->{online_api}",
|
||||
Content => {
|
||||
"source" => join(',', @{$ip}),
|
||||
"destination" => get_public_ip()
|
||||
}
|
||||
);
|
||||
unless ($response->is_success){
|
||||
die "an error occured while querying the API" .
|
||||
"The error is: " . $response->status_line;
|
||||
}
|
||||
}
|
||||
|
||||
# Get a VM configuration
|
||||
sub get_guest_conf {
|
||||
if ($ENV{PVE_GUEST_TYPE} eq 'lxc'){
|
||||
return PVE::LXC::Config->load_config(shift);
|
||||
} else {
|
||||
return PVE::QemuConfig->load_config(shift);
|
||||
}
|
||||
}
|
||||
|
||||
# Unplug and plug back nics connected on the WAN bridge
|
||||
# Needed when moving a failover IP to force flushing the ARP cache
|
||||
# in the guest as the gateway doesn't change, but its MAC does
|
||||
sub set_guest_nic_down_up {
|
||||
my $vmid = shift;
|
||||
my $vm_conf = get_guest_conf($vmid);
|
||||
my $nics = [];
|
||||
my @mod_nic = ();
|
||||
foreach my $key (keys %{$vm_conf}){
|
||||
# Only process netX elements, and which are connected on the WAN bridge
|
||||
next unless ($key =~ m/^net\d+/ && $vm_conf->{$key} =~ m/bridge=$conf->{wan_bridge}/);
|
||||
my $props = $vm_conf->{$key};
|
||||
next if ($props =~ m/link_down=0/);
|
||||
push @mod_nic, $key;
|
||||
if (defined &PVE::QemuServer::Monitor::hmp_cmd){
|
||||
PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $key off");
|
||||
} else {
|
||||
# PVE 5.x doesn't have PVE::QemuServer::Monitor::hmp_cmd
|
||||
PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $key off");
|
||||
}
|
||||
}
|
||||
sleep 1;
|
||||
foreach my $nic (@mod_nic){
|
||||
if (defined &PVE::QemuServer::Monitor::hmp_cmd){
|
||||
PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $nic on");
|
||||
} else {
|
||||
PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $nic on");
|
||||
}
|
||||
}
|
||||
}
|
||||
4
roles/pve/files/pve_dump
Normal file
4
roles/pve/files/pve_dump
Normal file
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
|
||||
tar cJf /home/lbkp/pve/pve-cluster.txz -C /var/lib pve-cluster
|
||||
tar cJf /home/lbkp/pve/pve-firewall.txz -C /var/lib pve-firewall
|
||||
3
roles/pve/files/pve_rm_dump
Normal file
3
roles/pve/files/pve_rm_dump
Normal file
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
rm -f /home/lbkp/pve/*
|
||||
11
roles/pve/files/remove_nag.patch
Normal file
11
roles/pve/files/remove_nag.patch
Normal file
@@ -0,0 +1,11 @@
|
||||
--- /usr/share/perl5/PVE/API2/Subscription.pm.orig 2018-11-22 09:29:26.612623539 +0100
|
||||
+++ /usr/share/perl5/PVE/API2/Subscription.pm 2018-11-22 09:30:07.639521319 +0100
|
||||
@@ -114,7 +114,7 @@
|
||||
my $info = PVE::INotify::read_file('subscription');
|
||||
if (!$info) {
|
||||
my $no_subscription_info = {
|
||||
- status => "NotFound",
|
||||
+ status => "Active",
|
||||
message => "There is no subscription key",
|
||||
url => $url,
|
||||
};
|
||||
65
roles/pve/files/unlock_dev
Executable file
65
roles/pve/files/unlock_dev
Executable file
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use JSON;
|
||||
use Term::ReadKey;
|
||||
use File::Which;
|
||||
|
||||
my $pvesh = which('pvesh');
|
||||
|
||||
# Are we using the new pvesh for which we have to specify the output format ?
|
||||
my $pvesh_opt = (system("$pvesh get /version --output-format=json >/dev/null 2>&1") == 0) ? '--output-format=json' : '';
|
||||
|
||||
# Get a list of every iSCSI storages defined on the cluster
|
||||
my $stor_iscsi = from_json(qx($pvesh get storage --type=iscsi $pvesh_opt 2>/dev/null));
|
||||
my @luks_dev = ();
|
||||
|
||||
# Now, check if it's encrypted using luks
|
||||
foreach my $stor (@{$stor_iscsi}){
|
||||
push @luks_dev, $stor if (is_luks(dev_from_stor($stor)));
|
||||
}
|
||||
|
||||
# If we have at least one device, we must ask for the password to unlock
|
||||
if (scalar @luks_dev gt 0){
|
||||
ReadMode( "noecho");
|
||||
print "Enter the password to unlock encrypted devices :";
|
||||
chomp (my $pwd = <>);
|
||||
print "\n";
|
||||
ReadMode ("original");
|
||||
foreach my $stor (@luks_dev){
|
||||
open $cmd,'|-', '/sbin/cryptsetup', 'open', '--type=luks', dev_from_stor($stor), $stor->{storage}, '--key-file=-';
|
||||
print $cmd $pwd;
|
||||
}
|
||||
}
|
||||
|
||||
# Return 1 if the device is a luks container
|
||||
sub is_luks {
|
||||
my $dev = shift;
|
||||
my $blkid = qx(/sbin/blkid $dev);
|
||||
my $type = 'unknown';
|
||||
if ($blkid =~ m/TYPE="(\w+)"/){
|
||||
$type = $1;
|
||||
}
|
||||
return ($type eq 'crypto_LUKS') ? 1 : 0;
|
||||
}
|
||||
|
||||
# Return the device node from the JSON storage object
|
||||
sub dev_from_stor {
|
||||
my $stor = shift;
|
||||
my $dev = '';
|
||||
if ($stor->{type} eq 'iscsi'){
|
||||
my $portal = ($stor->{portal} =~ m/:(\d+)$/) ? $stor->{portal} : $stor->{portal} . ':3260';
|
||||
$dev = '/dev/disk/by-path/ip-' . $portal . '-iscsi-' . $stor->{target} . '-lun-0';
|
||||
}
|
||||
return $dev;
|
||||
}
|
||||
|
||||
# If ocfs2 is used, o2cb must be restarted as it's started too early to setup everything correctly
|
||||
#if (-e '/etc/init.d/o2cb'){
|
||||
# print "Restarting o2cb and mounting other filesystems";
|
||||
# system('/bin/systemctl', 'restart', 'o2cb');
|
||||
# sleep 20;
|
||||
# system('/bin/mount', '-a');
|
||||
# # Not sure why but OCFS2 seems to fail on first mount
|
||||
# system('/bin/mount', '-a');
|
||||
# print "\n";
|
||||
#}
|
||||
Reference in New Issue
Block a user