Update to 2021-12-01 19:13

roles/pve/defaults/main.yml

@@ -0,0 +1,36 @@
+---
+
+pve_dump_storage: local
+
+pve_mod_to_load:
+  - nf_conntrack_proto_gre
+
+pve_ksm: True
+pve_ksm_sleep_msec: 5
+pve_ksm_thres_coef: 35
+pve_ksm_thres_const: 8000
+pve_ksm_npages_boost: 1200
+
+pve_ocfs2: False
+pve_ocfs2_heartbeat_thres: 8
+
+pve_online: False
+# pve_online_api_key:
+pve_wan_bridge: vmbr1
+#pve_vlan_bridge: vmbr0
+# If true, the hook script on migrate will unplug/replug the WAN nic
+# to force an ARP flush in the guest.
+# This is not needed if the wan bridge has the same MAC addr on every node
+pve_migrate_flush_arp: True
+
+pve_ovh: False
+
+pve_cluster_vhosts: []
+# pve_cluster_vhosts:
+#   - pve.example.net
+#   - pve.example.com
+pve_letsencrypt: False
+
+# If enabled, a cron task will populate zabbix monitoring cache
+# needs zabbix_agent to be deployed too
+pve_zabbix_cache: False

roles/pve/files/online_hook.pl

@@ -0,0 +1,111 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+use File::Basename;
+use JSON;
+use Logger::Syslog;
+
+$SIG{CHLD} = 'IGNORE';
+
+info( "GUEST HOOK: " . join( ' ', @ARGV ) );
+
+my $vmid = shift;
+my $phase = shift;
+
+if ( not -l '/etc/pve/local' ) {
+  error( "Can't find /etc/pve/local link" );
+  die;
+} elsif ( not defined $vmid or not defined $phase ) {
+  error( "Need to pass both vmid and phase arguments" );
+  die;
+} elsif ( $vmid !~ m/^\d+$/ ) {
+  error( "vmid must be only numerical" );
+  die;
+}
+
+# We must identify the local node
+my $local_node = basename( readlink( '/etc/pve/local' ) );
+
+if ( $phase eq 'pre-start' ) {
+  info( "Runing pre-start hook for guest $vmid" );
+} elsif ( $phase eq 'post-start' ) {
+  info( "Running post-start hook for guest $vmid" );
+  # A VM has just started. Let's check if it's already on the local node
+  # If it's an incoming live migration, it might still be running on another node
+  # All of this must run in the background because hooks are synchronous
+  my $pid = fork();
+  if ( $pid != 0 ) {
+    # main script can stop now, everything will run in a forked process
+    POSIX::_exit 0;
+  } elsif ( defined $pid ) {
+    # All those fh must be closed for the fork to be independant of its parent
+    close STDOUT;
+    close STDERR;
+    close STDIN;
+    POSIX::setsid();
+
+    my $i = 0;
+    my $guest_found = 0;
+
+    # We'll loop for up to 30min, which should be sufficient. If migration takes longer than that,
+    # something is probably wrong
+    LOOP: while ( $i lt 1800 ) {
+      # Here, we query the API for all the VM
+      my $resources = from_json( qx(pvesh get /cluster/resources --type=vm --output-format=json) );
+      # Then we loop through all the VM to find the one we're interested in
+      foreach my $vm ( @{$resources} ){
+        next if ( $vm->{id} !~ m{^(qemu|lxc)/$vmid$} );
+
+        # OK, we found the guest $vmid
+        info("Found guest $vmid, running on node " . $vm->{node});
+        $guest_found = 1;
+
+        # Is the guest running on local node ? If yes, it means migration is finished, and we
+        # can redirect IP failover and routing table
+        if ( $vm->{node} eq $local_node ) {
+
+          # pve-online use this env var to check if we must unplug/replug the WAN NIC
+          $ENV{PVE_GUEST_TYPE} = $1;
+
+          # And here we go !
+          qx(/bin/systemd-cat /usr/local/bin/pve-online --update-routes --update-gre --migrate-ipfo=$vmid);
+
+          # Update routing table of the other online nodes
+          my $nodes = from_json( qx(pvesh get /nodes --output-format=json) );
+          foreach my $node ( @{$nodes} ) {
+            if ( $node->{status} eq 'online' and $node->{node} ne $local_node ) {
+              info("Updating routing table of node $node->{node}");
+              qx(ssh -o ConnectTimeout=3 -l root $node->{node} /usr/local/bin/pve-online --update-routes);
+            }
+          }
+
+          # And we're done, stop looping
+          last LOOP;
+
+        # Guest is not running on the local node = migration is still running
+        # Wait a bit and start again
+        } else {
+          info( "Guest $vmid is still running on node " . $vm->{node} . " not yet on $local_node. Waiting a bit more for migration to finish" );
+          sleep 1;
+          next LOOP;
+        }
+      }
+
+      # We looped through all the guests and couldn't find the one we're looking for, nothing more we can do
+      if ( not $guest_found ) {
+        error( "No such guest with id $vmid" );
+        die;
+      }
+    }
+  }
+} elsif ( $phase eq 'pre-stop' ) {
+  info( "Running pre-stop hook" );
+} elsif ( $phase eq 'post-stop' ) {
+  info( "Running post-stop hook" );
+  # Just remove routes if needed
+  qx(/bin/systemd-cat /usr/local/bin/pve-online --update-routes)
+} else {
+  error( "Unknown hook phase : $phase" );
+  die;
+}

roles/pve/files/pve-online

@@ -0,0 +1,328 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+use Getopt::Long;
+use Config::Simple;
+use LWP::UserAgent;
+use JSON;
+use PVE::QemuServer;
+use PVE::LXC::Config;
+use Array::Diff;
+use Net::Address::IP::Local;
+use Net::Route::Table;
+use NetAddr::IP;
+use Data::Validate::IP qw(is_ipv4);
+use Data::Dumper;
+
+$| = 1;
+
+my $config = '/etc/pve-online.conf';
+my ($update_routes, $update_gre, $migrate_ipfo) = undef;
+
+GetOptions(
+  "config=s"       => \$config,
+  "update-routes"  => \$update_routes,
+  "update-gre"     => \$update_gre,
+  "migrate-ipfo=i" => \$migrate_ipfo
+);
+
+# Config can be stored in /etc/pve
+# Lets wait a bit for it to be available
+if ($config =~ m|^/etc/pve|){
+  my $t = 0;
+  while ($t < 120){
+    if (!-e $config){
+      print "$config not yet available\n";
+      sleep 2;
+      $t += 2;
+    } else {
+      last;
+    }
+  }
+}
+if (!-e $config){
+  die "$config doesn't exist\n";
+}
+
+my $cfg = new Config::Simple;
+$cfg->read($config);
+my $conf = $cfg->get_block('general');
+my $lwp = new LWP::UserAgent;
+my $online_id = undef;
+
+if (!$conf){
+  die "No general section found in $config\n";
+}
+if (!$conf->{online_api}){
+  die "No online_api defined in $config\n";
+}
+
+# Set some defaults
+$conf->{wan_bridge}  ||= 'vmbr1';
+$conf->{migrate_flush_arp} ||= 'yes';
+
+if ($update_routes){
+  update_routes();
+}
+if ($update_gre){
+  if (defined $conf->{vlan_bridge}){
+    update_gre();
+  } else {
+    print "No VLAN bridge defined. Don't setup GRE tunnels\n";
+  }
+}
+if ($migrate_ipfo){
+  my $ipfo = [];
+  # We parse the description field which contains the list of ipfo
+  # attached to this VM
+  foreach my $line ( split("\n", get_guest_conf($migrate_ipfo)->{description} || "") ){
+    if ($line =~ m/^\s*ipfo\d*:\s+(\d+\.\d+\.\d+\.\d+)/){
+      my $candidate = $1;
+      if (is_ipv4($candidate)){
+        push @{$ipfo}, $candidate;
+        print "Found IP $candidate assigned to guest $migrate_ipfo\n";
+      } else {
+        print "Found $candidate assigned to guest $migrate_ipfo which doesn't look like a valid IP\n";
+      }
+    }
+  }
+  $online_id ||= get_online_id();
+  # Now we check if we need to migrate IPFO on the local server
+  my $ipfo_diff = Array::Diff->diff(get_ip_fo($online_id), $ipfo);
+  if (scalar @{$ipfo_diff->added} > 0){
+    print "Update needed. " . join(' ', @{$ipfo_diff->added}) . " should be redirected on server $online_id\n";
+    redirect_ipfo($ipfo_diff->added);
+    update_routes();
+    if ( $ENV{PVE_GUEST_TYPE} ne 'lxc' and $conf->{migrate_flush_arp} =~ m/^1|yes|true|on$/i ){
+      set_guest_nic_down_up($migrate_ipfo);
+    }
+  }
+}
+
+#####################################
+# Sub routines
+#####################################
+
+# Query Online's API
+sub query_online_api{
+  my $uri = shift;
+  $uri = '/api/v1' . $uri if ($uri !~ m|^/api/v1|);
+  my $response = $lwp->get("https://api.online.net$uri",
+    "Authorization" => "Authorization: Bearer $conf->{online_api}",
+  );
+  unless ($response->is_success){
+    die "an error occured while querying the API" .
+        "The error is: " . $response->status_line;
+  }
+  return from_json($response->content);
+}
+
+# Update routes for ARP Proxy
+sub update_routes {
+  $online_id ||= get_online_id();
+  # This is the list of IP which we should have
+  # on the routing table
+  my $routes_online = get_ip_fo($online_id);
+
+  # This is the actual list of IP for which we have routes
+  my $routes_local = get_local_routes();
+
+  # Now, we have to remove routes for those in $routes_local but not in $routes_online
+  # And add routes for those in $routes_online but not in $routes_local
+  my $diff = Array::Diff->diff($routes_online, $routes_local);
+  foreach my $route (@{$diff->added}){
+    next if (grep { $_ eq $route } @{$diff->deleted});
+    print "Removing route for $route\n";
+    system(
+      '/sbin/ip',
+      'route',
+      'del',
+      $route,
+      'dev',
+      $conf->{wan_bridge}
+    );
+  }
+  foreach my $route (@{$diff->deleted}){
+    next if (grep { $_ eq $route } @{$diff->added});
+    print "Adding route for $route\n";
+    system(
+      '/sbin/ip',
+      'route',
+      'add',
+      $route . '/32',
+      'dev',
+      $conf->{wan_bridge}
+    );
+  }
+}
+
+# Get the list of routes defined on $conf->{wan_bridge}
+sub get_local_routes {
+  my $ip = [];
+  my $routes = Net::Route::Table->from_system();
+  foreach my $route (@{$routes->all_routes()}){
+    if ($route->{interface} eq $conf->{wan_bridge} and $route->destination()->masklen() == 32){
+      push @{$ip}, $route->destination()->addr();
+    }
+  }
+  return $ip;
+}
+
+# Get the list of IP failover assigned to a server. Taks a server ID as only arg
+sub get_ip_fo {
+  my $srv_id = shift;
+  return get_srv_info($srv_id)->{network}->{ipfo};
+}
+
+# Return server info
+sub get_srv_info {
+  my $srv_id = shift;
+  return query_online_api('/server/info/' . $srv_id);
+}
+
+# Return this server's public IP
+sub get_public_ip {
+  return Net::Address::IP::Local->public_ipv4;
+}
+
+# Return Online's server id
+sub get_online_id {
+  if (-e '/tmp/online_id'){
+    open my $id_file, '</tmp/online_id';
+    $online_id = <$id_file>;
+    close $id_file;
+  } else {
+    my $ip = get_public_ip();
+    foreach my $srv (@{query_online_api('/server')}){
+      my $info = query_online_api($srv);
+      if ($info->{network}->{ip}[0] eq $ip){
+        $online_id = $info->{id};
+        last;
+      }
+    }
+    open my $id_file, ">/tmp/online_id";
+    print $id_file $online_id;
+    close $id_file;
+  }
+  print "My Online's ID is $online_id\n";
+  return $online_id;
+}
+
+sub update_gre {
+  # We have to setup GRE tunnels with all other members
+  # to connect $conf->{vlan_bridge} between every nodes
+  # Something like
+  # ovs-vsctl add-port vmbr1 gre0 -- set interface gre0 type=gre options:remote_ip=''10.29.254.2''
+  # We just have to automate this for every nodes of the cluster
+  print "Getting cluster status...\n";
+  my $members = get_cluster_members();
+  print "Found " . scalar @{$members} . " members\n";
+  my $gre = 0;
+
+  print "Counting GRE ports...\n";
+  my @ports = qx(ovs-vsctl list-ports $conf->{vlan_bridge});
+  my @gre_ports = grep { $_ =~ m/^gre/ } @ports;
+  print "Found " . scalar @gre_ports . " GRE ports\n";
+
+  if (scalar @gre_ports ne scalar @{$members} - 1){
+    print "We need to update GRE tunnels\n";
+    # Remove all greX ports from the VLAN bridge
+    foreach my $port ( @ports ){
+      chomp($port);
+      next unless ($port =~ m/^gre\d+$/);
+      print "Removing port $port from $conf->{vlan_bridge}\n";
+      system(
+        'ovs-vsctl',
+        'del-port',
+        $conf->{vlan_bridge},
+        $port
+      );
+    }
+    # And setup one GRE tunnel per node
+    foreach my $member (@{$members}){
+      # We must skip our own node
+      if (Net::Address::IP::Local->connected_to($member) ne $member){
+        print "Adding GRE interface gre$gre to tunnel with $member\n";
+        system(
+          'ovs-vsctl',
+          'add-port',
+          $conf->{vlan_bridge},
+          'gre' . $gre,
+          '--',
+          'set',
+          'interface',
+          'gre' . $gre,
+          'type=gre',
+          'options:remote_ip=' . $member
+        );
+        $gre++;
+      }
+    }
+  }
+}
+
+# Get the list of members of this proxmox cluster
+sub get_cluster_members {
+  my $ip = [];
+  foreach my $line (qx(corosync-cmapctl)){
+    push @{$ip}, $1 if ($line =~ m/ip\((\d+\.\d+\.\d+\.\d+)\)/);
+  }
+  return $ip;
+}
+
+sub redirect_ipfo {
+  my $ip = shift;
+  print "Redirecting failover IP " . join(', ', @{$ip}) . "\n";
+  my $response = $lwp->post("https://api.online.net/api/v1/server/failover/edit",
+    Authorization => "Authorization: Bearer $conf->{online_api}",
+    Content => {
+      "source"        => join(',', @{$ip}),
+      "destination"   => get_public_ip()
+    }
+  );
+  unless ($response->is_success){
+    die "an error occured while querying the API" .
+        "The error is: " . $response->status_line;
+  }
+}
+
+# Get a VM configuration
+sub get_guest_conf {
+  if ($ENV{PVE_GUEST_TYPE} eq 'lxc'){
+    return PVE::LXC::Config->load_config(shift);
+  } else {
+    return PVE::QemuConfig->load_config(shift);
+  }
+}
+
+# Unplug and plug back nics connected on the WAN bridge
+# Needed when moving a failover IP to force flushing the ARP cache
+# in the guest as the gateway doesn't change, but its MAC does
+sub set_guest_nic_down_up {
+  my $vmid = shift;
+  my $vm_conf = get_guest_conf($vmid);
+  my $nics = [];
+  my @mod_nic = ();
+  foreach my $key (keys %{$vm_conf}){
+    # Only process netX elements, and which are connected on the WAN bridge
+    next unless ($key =~ m/^net\d+/ && $vm_conf->{$key} =~ m/bridge=$conf->{wan_bridge}/);
+    my $props = $vm_conf->{$key};
+    next if ($props =~ m/link_down=0/);
+    push @mod_nic, $key;
+    if (defined &PVE::QemuServer::Monitor::hmp_cmd){
+      PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $key off");
+    } else {
+      # PVE 5.x doesn't have PVE::QemuServer::Monitor::hmp_cmd
+      PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $key off");
+    }
+  }
+  sleep 1;
+  foreach my $nic (@mod_nic){
+    if (defined &PVE::QemuServer::Monitor::hmp_cmd){
+      PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $nic on");
+    } else {
+      PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $nic on");
+    }
+  }
+}

roles/pve/files/pve_dump

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+tar cJf /home/lbkp/pve/pve-cluster.txz -C /var/lib pve-cluster
+tar cJf /home/lbkp/pve/pve-firewall.txz -C /var/lib pve-firewall

roles/pve/files/pve_rm_dump

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+rm -f /home/lbkp/pve/*

roles/pve/files/remove_nag.patch

@@ -0,0 +1,11 @@
+--- /usr/share/perl5/PVE/API2/Subscription.pm.orig	2018-11-22 09:29:26.612623539 +0100
++++ /usr/share/perl5/PVE/API2/Subscription.pm	2018-11-22 09:30:07.639521319 +0100
+@@ -114,7 +114,7 @@
+ 	my $info = PVE::INotify::read_file('subscription');
+ 	if (!$info) {
+ 	    my $no_subscription_info = {
+-		status => "NotFound",
++		status => "Active",
+ 		message => "There is no subscription key",
+ 		url => $url,
+ 	    };

roles/pve/files/unlock_dev

@@ -0,0 +1,65 @@
+#!/usr/bin/perl -w
+
+use JSON;
+use Term::ReadKey;
+use File::Which;
+
+my $pvesh = which('pvesh');
+
+# Are we using the new pvesh for which we have to specify the output format ?
+my $pvesh_opt = (system("$pvesh get /version --output-format=json >/dev/null 2>&1") == 0) ? '--output-format=json' : '';
+
+# Get a list of every iSCSI storages defined on the cluster
+my $stor_iscsi = from_json(qx($pvesh get storage --type=iscsi $pvesh_opt 2>/dev/null));
+my @luks_dev = ();
+
+# Now, check if it's encrypted using luks
+foreach my $stor (@{$stor_iscsi}){
+  push @luks_dev, $stor if (is_luks(dev_from_stor($stor)));
+}
+
+# If we have at least one device, we must ask for the password to unlock
+if (scalar @luks_dev gt 0){   
+  ReadMode( "noecho");
+  print "Enter the password to unlock encrypted devices :";
+  chomp (my $pwd = <>);
+  print "\n";
+  ReadMode ("original");
+  foreach my $stor (@luks_dev){
+    open $cmd,'|-', '/sbin/cryptsetup', 'open', '--type=luks', dev_from_stor($stor), $stor->{storage}, '--key-file=-';
+    print $cmd $pwd;
+  }
+}
+
+# Return 1 if the device is a luks container
+sub is_luks {
+  my $dev = shift;
+  my $blkid = qx(/sbin/blkid $dev);
+  my $type = 'unknown';
+  if ($blkid =~ m/TYPE="(\w+)"/){
+    $type = $1;
+  }
+  return ($type eq 'crypto_LUKS') ? 1 : 0;
+}
+
+# Return the device node from the JSON storage object
+sub dev_from_stor {
+  my $stor = shift;
+  my $dev = '';
+  if ($stor->{type} eq 'iscsi'){
+    my $portal = ($stor->{portal} =~ m/:(\d+)$/) ? $stor->{portal} : $stor->{portal} . ':3260';
+    $dev = '/dev/disk/by-path/ip-' . $portal . '-iscsi-' . $stor->{target} . '-lun-0';
+  }
+  return $dev;
+}
+
+# If ocfs2 is used, o2cb must be restarted as it's started too early to setup everything correctly
+#if (-e '/etc/init.d/o2cb'){
+#  print "Restarting o2cb and mounting other filesystems";
+#  system('/bin/systemctl', 'restart', 'o2cb');
+#  sleep 20;
+#  system('/bin/mount', '-a');
+#  # Not sure why but OCFS2 seems to fail on first mount
+#  system('/bin/mount', '-a');
+#  print "\n";
+#}

roles/pve/handlers/main.yml

@@ -0,0 +1,19 @@
+---
+
+- include: ../common/handlers/main.yml
+
+- name: restart ksmtuned
+  service: name=ksmtuned state=restarted
+
+- name: restart o2cb
+  service: name=o2cb state=restarted
+  when: pve_ocfs2 == True
+
+- name: restart pveproxy
+  service: name=pveproxy state=restarted
+
+- name: restart pvedaemon
+  service: name=pvedaemon state=restarted
+
+- name: restart pve-ha-lrm
+  service: name=pve-ha-lrm state=restarted

roles/pve/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - role: letsencrypt
+    when: pve_letsencrypt is defined and pve_letsencrypt
+  - role: zfs_common

roles/pve/tasks/facts.yml

@@ -0,0 +1,7 @@
+---
+
+  # The module is unavailable
+- name: Do not load nf_conntrack_proto_gre for PVE6
+  set_fact: pve_mod_to_load={{ pve_mod_to_load | difference(['nf_conntrack_proto_gre']) }}
+  when: ansible_distribution_major_version | int >= 10
+  tags: pve

roles/pve/tasks/filebeat.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Deploy filebeat configuration
+  template: src=filebeat.yml.j2 dest=/etc/filebeat/ansible_inputs.d/pve.yml
+  tags: pve,log

roles/pve/tasks/main.yml

@@ -0,0 +1,165 @@
+---
+
+- include: facts.yml
+
+- name: Install tools
+  apt:
+    name:
+      - pigz
+      - ksm-control-daemon
+      - openvswitch-switch
+      - ethtool
+      - patch
+
+- name: Deploy vzdump config
+  template: src=vzdump.conf.j2 dest=/etc/vzdump.conf
+
+- name: Deploy ksm configuration
+  template: src=ksmtuned.conf.j2 dest=/etc/ksmtuned.conf
+  notify: restart ksmtuned
+
+- name: Handle ksm services
+  service: name=ksmtuned state={{ pve_ksm | ternary('started','stopped') }} enabled={{ pve_ksm | ternary(True,False) }}
+
+- name: Configure modules to load
+  copy: content={{ pve_mod_to_load | join("\n") }} dest=/etc/modules-load.d/firewall.conf
+  register: pve_modules
+
+- name: Load modules
+  service: name=systemd-modules-load state=restarted
+  when: pve_modules.changed
+
+- name: Enable ocfs2 support
+  set_fact: pve_ocfs2=True
+  when: fstab | default([]) | selectattr('fstype','equalto','ocfs2') | list | length > 0
+
+- name: Install ocfs2 support
+  apt: name=ocfs2-tools state=present
+  when: pve_ocfs2 == True
+
+- name: Check proxmox cluster status
+  command: pvesh get /cluster/status --output-format=json
+  register: pve_cluster_status_1
+  ignore_errors: True
+  changed_when: False
+
+- name: Parse proxmox cluster status
+  set_fact: pve_cluster={{ pve_cluster_status_1.stdout | from_json }}
+  when: pve_cluster_status_1.rc == 0
+
+- name: Check proxmox cluster status (old pvesh)
+  command: pvesh get /cluster/status
+  when: pve_cluster_status_1.rc != 0
+  register: pve_cluster_status_2
+  changed_when: False
+
+- name: Parse proxmox cluster status (old pvesh)
+  set_fact: pve_cluster={{ pve_cluster_status_2.stdout | from_json }}
+  when: pve_cluster_status_1.rc != 0
+
+- name: Deploy ocfs2 config
+  template: src=ocfs2.conf.j2 dest=/etc/ocfs2/cluster.conf
+  when: pve_ocfs2 == True
+  notify: restart o2cb
+
+- name: Deploy o2cb conf
+  template: src=o2cb.j2 dest=/etc/default/o2cb
+  when: pve_ocfs2 == True
+
+- name: Handle o2cb service
+  service: name=o2cb state=started enabled=True
+  when: pve_ocfs2 == True
+
+- name: Deploy the unlock_dev script
+  copy: src=unlock_dev dest=/usr/local/bin/unlock_dev mode=755
+
+- name: Check if the old hookd daemon is installed
+  stat: path=/usr/local/bin/pve-hookd
+  register: pve_old_hookd
+
+- name: Stop the old hookd daemon
+  service: name=pve-hookd state=stopped
+  when: pve_old_hookd.stat.exists
+
+- name: Remove the old hook daemon
+  file: path={{ item }} state=absent
+  loop:
+    - /usr/local/bin/pve-hookd
+    - /etc/hooks
+    - /etc/systemd/system/pve-hookd.service
+    - /etc/tmpfiles.d/pve-container-hooks.conf
+    - /etc/systemd/system/pve-container@.service.d/pve-container-hooks.conf
+    - /var/run/lxc/active
+
+- name: Reload systemd
+  command: systemctl daemon-reload
+  when: pve_old_hookd.stat.exists
+
+- include_tasks: pve_online.yml
+  when: pve_online == True
+
+- include_tasks: ovh.yml
+  when: pve_ovh == True
+
+- name: Create backup dir
+  file: path=/home/lbkp/pve state=directory
+
+- name: Install pre and post backup scripts
+  copy: src={{ item.src }} dest=/etc/backup/{{ item.type }}.d/{{ item.src }} mode=755
+  with_items:
+    - src: pve_dump
+      type: pre
+    - src: pve_rm_dump
+      type: post
+
+- name: Remove registration nag
+  patch: src=remove_nag.patch dest=/usr/share/perl5/PVE/API2/Subscription.pm
+  ignore_errors: True # Don't fail on old PVE where the patch doesn't apply
+  notify: restart pveproxy
+
+- name: Rise limits for containers
+  pam_limits:
+    domain: '*'
+    limit_type: "{{ item.type }}"
+    limit_item: nofile
+    value: "{{ item.value }}"
+  with_items:
+    - type: soft
+      value: 65000
+    - type: hard
+      value: 65535
+
+- name: Rise inotify instances
+  sysctl:
+    name: fs.inotify.max_user_instances
+    value: 1024
+    sysctl_file: /etc/sysctl.d/ansible.conf
+
+- name: Ensure dehydrated hook dir exists
+  file: path=/etc/dehydrated/hooks_deploy_cert.d/ state=directory
+
+- name: Deploy dehydrated hook
+  template: src=dehydrated_hook.sh.j2 dest=/etc/dehydrated/hooks_deploy_cert.d/20pve.sh mode=755
+
+  # See https://bugzilla.proxmox.com/show_bug.cgi?id=2326 why
+- name: Create corosync override directory
+  file: path=/etc/systemd/system/corosync.service.d/ state=directory
+  tags: pve
+
+- name: Setup corosync to be restarted in case of failure
+  copy:
+    content: |
+      [Service]
+      Restart=on-failure
+      RestartSec=1
+    dest: /etc/systemd/system/corosync.service.d/ansible.conf
+  register: pve_corosync_unit
+  tags: pve
+
+- name: Reload systemd
+  systemd: daemon_reload=True
+  when: pve_corosync_unit.changed
+  tags: pve
+
+- include: zabbix.yml
+- include: filebeat.yml

roles/pve/tasks/ovh.yml

@@ -0,0 +1,6 @@
+---
+
+- name: Add vrack routing table
+  copy:
+    content: "1 vrack"
+    dest: /etc/iproute2/rt_tables.d/ovh.conf

roles/pve/tasks/pve_online.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Install dependencies
+  apt:
+    name:
+      - libnet-route-perl
+      - libnet-address-ip-local-perl
+      - libarray-diff-perl
+      - libdata-validate-ip-perl
+      - liblogger-syslog-perl
+
+- name: Deploy Online.net integration script
+  copy: src=pve-online dest=/usr/local/bin/pve-online mode=755
+
+- name: Deploy Online.net integration conf
+  template: src=pve-online.conf.j2 dest=/etc/pve-online.conf mode=600
+
+- name: Create hook directory on local storage
+  file: path=/var/lib/vz/snippets state=directory
+
+- name: Deploy Online hook
+  copy: src=online_hook.pl dest=/var/lib/vz/snippets/online_hook.pl mode=755
+
+- name: Ensure /etc/systemd/system/ exists
+  file: path=/etc/systemd/system/ state=directory
+
+- name: Disable pve-online-gre service
+  service: name=pve-online-gre state=stopped enabled=False
+  failed_when: False
+
+- name: Remove pve-online-gre service unit
+  file: path=/etc/systemd/system/pve-online-gre.service state=absent
+  register: pve_gre_unit
+
+- name: Reload systemd
+  systemd: daemon_reload=True
+  when: pve_gre_unit.changed
+

roles/pve/tasks/zabbix.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Check if check_pve_sudo is installed
+  stat: path=/var/lib/zabbix/bin/util_populate_pve_cache
+  register: pve_zabbix_scripts
+  tags: pve,zabbix
+
+- name: Setup a cron job for Zabbix monitoring cache population
+  cron:
+    name: pve_zabbix_cache
+    cron_file: pve_zabbix_cache
+    user: root
+    job: "/var/lib/zabbix/bin/util_populate_pve_cache"
+    minute: "*/5"
+    state: "{{ (pve_zabbix_cache and pve_zabbix_scripts.stat.exists) | ternary('present','absent') }}"
+  tags: pve,zabbix

roles/pve/templates/dehydrated_hook.sh.j2

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+{% if pve_letsencrypt %}
+if [ $1 == "{{ inventory_hostname }}" ]; then
+  cat /var/lib/dehydrated/certificates/certs/{{ inventory_hostname }}/privkey.pem > /etc/pve/local/pveproxy-ssl.key
+  cat /var/lib/dehydrated/certificates/certs/{{ inventory_hostname }}/fullchain.pem > /etc/pve/local/pveproxy-ssl.pem
+  chown root:www-data /etc/pve/local/pveproxy-ssl.{key,pem}
+  chmod 640 /etc/pve/local/pveproxy-ssl.{key,pem}
+  /bin/systemctl reload pveproxy
+fi
+{% endif %}

roles/pve/templates/filebeat.yml.j2

@@ -0,0 +1,11 @@
+- type: log
+  enabled: True
+  paths:
+    - /var/log/pve-*.log
+    - /var/log/pveam.log
+    - /var/log/pveproxy/access.log
+    - /var/log/openvswitch/*.log
+    - /var/log/vzdump/*.log
+  exclude_files:
+    - '\.[gx]z$'
+    - '\d+$'

roles/pve/templates/ksmtuned.conf.j2

@@ -0,0 +1,6 @@
+KSM_SLEEP_MSEC={{ pve_ksm_sleep_msec }}
+KSM_THRES_COEF={{ pve_ksm_thres_coef }}
+KSM_THRES_CONST={{ pve_ksm_thres_const }}
+KSM_NPAGES_BOOST={{ pve_ksm_npages_boost }}
+KSM_NPAGES_DECAY=-200
+KSM_NPAGES_MAX=4800

roles/pve/templates/o2cb.j2

@@ -0,0 +1,17 @@
+# O2CB_ENABLED: 'true' means to load the driver on boot.
+O2CB_ENABLED=true
+
+# O2CB_BOOTCLUSTER: If not empty, the name of a cluster to start.
+O2CB_BOOTCLUSTER=ocfs2
+
+# O2CB_HEARTBEAT_THRESHOLD: Iterations before a node is considered dead.
+O2CB_HEARTBEAT_THRESHOLD={{ pve_ocfs2_heartbeat_thres }}
+
+# O2CB_IDLE_TIMEOUT_MS: Time in ms before a network connection is considered dead.
+O2CB_IDLE_TIMEOUT_MS=30000
+
+# O2CB_KEEPALIVE_DELAY_MS: Max. time in ms before a keepalive packet is sent.
+O2CB_KEEPALIVE_DELAY_MS=2000
+
+# O2CB_RECONNECT_DELAY_MS: Min. time in ms between connection attempts.
+O2CB_RECONNECT_DELAY_MS=2000

roles/pve/templates/ocfs2.conf.j2

@@ -0,0 +1,12 @@
+{% for element in pve_cluster | selectattr("type","equalto","node") | sort(attribute='id')%}
+node:
+  ip_port = 7777
+  ip_address = {{ element.ip }}
+  number = {{ loop.index0 }}
+  name = {{ element.name }}
+  cluster = ocfs2
+
+{% endfor %}
+cluster:
+  node_count = {{ pve_cluster | selectattr("type","equalto","node") | list | length }}
+  name = ocfs2

roles/pve/templates/pve-hookd.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=PVE Hook daemon
+After=syslog.target network.target pveproxy.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/pve-hookd --hookdir /etc/hooks
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

roles/pve/templates/pve-online-gre.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Setup GRE tunnels with other PVE nodes
+After=pve-guests.service
+Requires=pve-cluster.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/bin/pve-online --update-gre
+
+[Install]
+WantedBy=multi-user.target

roles/pve/templates/pve-online.conf.j2

@@ -0,0 +1,7 @@
+[general]
+online_api  = {{ pve_online_api_key }}
+wan_bridge = {{ pve_wan_bridge }}
+{% if pve_vlan_bridge is defined %}
+vlan_bridge = {{ pve_vlan_bridge }}
+{% endif %}
+migrate_flush_arp = {{ (pve_migrate_flush_arp) | ternary('yes','no') }}

roles/pve/templates/vzdump.conf.j2

@@ -0,0 +1,6 @@
+mode: snapshot
+compress: 1
+pigz: 1
+mailnotification: failure
+mailto: {{ system_admin_email }}
+storage: {{ pve_dump_storage }}

roles/pve/vars/main.yml

@@ -0,0 +1,10 @@
+---
+# Never manage iptables, PVE provides its own daemon
+iptables_manage: False
+
+# We most likely want to enable ip_forwarding
+net_ipv4_forward: True
+
+# root must be able to connect with ssh, and without DNS
+sshd_permit_root_login: True
+sshd_use_dns: False