first cut web deploy

install-koji-farm.sh

@@ -2,11 +2,18 @@
 
 set -e
 DEBUG=
-for param in $1 $2 $3 ; do
+KOJI_HUB_FQDN="$(hostname -f)"
+KOJI_WEB_FQDN=$KOJI_HUB_FQDN
+KOJI_Build_FQDN=
+for param in $1 $2 $3 $4 $5 $6 $7; do
   if [ $param ] ; then
     case $param in
 	    debug )
 		    DEBUG="debug" ;;
+        web=* )
+            KOJI_WEB_FQDN=${param#=*} ;;
+        build=* )
+            KOJI_BUILD_FQDN=$KOJI_BUILD_FQDN" "${param#=*} ;;    
     esac
   else
 	  break
@@ -27,6 +34,7 @@ SCRIPT_DIR="$(echo ~)/bin"
 
 mkdir -p $SCRIPT_DIR
 curl $SILENT $SCRIPT_GIT/koji-deploy-hub.sh > $SCRIPT_DIR/koji-deploy-hub.sh
+curl $SILENT $SCRIPT_GIT/koji-deploy-web.sh > $SCRIPT_DIR/koji-deploy-web.sh
 curl $SILENT $SCRIPT_GIT/koji-add-builder.sh > $SCRIPT_DIR/koji-add-builder.sh
 curl $SILENT $SCRIPT_GIT/koji-bootstrap-build.sh > $SCRIPT_DIR/koji-bootstrap-build.sh
 curl $SILENT $SCRIPT_GIT/koji-add-user.sh > $SCRIPT_DIR/koji-add-user.sh
@@ -68,11 +76,10 @@ while true ; do
 done
 
 # setup default parameters
-KOJI_HUB_FQDN="$(hostname -f)"
 HTTPD_USER=apache
 HTTPD_DOCUMENT_ROOT=/var/www/html
 KOJI_DIR="/mnt/koji"
-TAG_NAME="smeos"
+TAG_NAME="smeos11,amecontribs11"
 KOJI_PKI_DIR=/etc/pki/koji
 RPM_ARCH="x86_64 noarch"
 KOJID_CAPACITY=16
@@ -119,23 +126,92 @@ chmod o+x $SCRIPT_DIR/koji-parameters.sh
 # deploy the central koji components
 koji-deploy-hub.sh $DEBUG
 
+# deploy the web koji components
+if [[ $KOJI_WEB_FQDN = $KOJI_HUB_FQDN ]] ; then
+    # deploy locally (default)
+    koji_deploy_web.sh $KOJI_WEB_FQDN $DEBUG
+else
+    # deploy remotely to $KOJI_WEB_FQDN
+    # check that I can conmnect
+    if [ ! nc -z $KOJI_WEB_FQDN 22 2>/dev/null ] ; then
+        echo "I cannot connect to $KOJI_WEB_FQDN! Is it online? "
+        echo "Options:"
+        echo "- turn on the server"
+        echo "- add this server into the /etc/hosts file on this server"
+        exit 1
+    fi    
+    # generate a hub ssh key if there isn't one already (for scp & ssh to web server)
+    if [ ! -f /root/.ssh/id-rsa ] ; then
+        # create a ssh key on build server
+        mkdir -p ~/.ssh
+        ssh-keygen -t rsa -f /root/.ssh/id_rsa -N ""
+    fi
+    # copy the server key into authorized keys on the web server
+    ssh-copy-id -i ~/.ssh/id_rsa.pub $KOJI_WEB_FQDN
+    # find the IP of the web server
+    WEB_IP=$(ssh root@$KOJI_WEB_FQDN "hostname -I")
+    # add web server into allowed access to db
+    cat >> "$POSTGRES_DIR"/data/pg_hba.conf <<- EOF
+host     koji        koji        $WEB_IP/32       scram-sha-256
+EOF
+    systemctl reload postgresql
+    # copy across the ssl keys
+    ssh $QUIET root@$KOJI_WEB_FQDN mkdir -p $KOJI_PKI_DIR/private
+    scp $QUIET $KOJI_PKI_DIR/$KOJI_WEB_FQDN.pem root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/.
+    scp $QUIET $KOJI_PKI_DIR/kojihub.pem root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/.
+    scp $QUIET $KOJI_PKI_DIR/private/kojihub.key root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/private/.
+    scp $QUIET $KOJI_PKI_DIR/koji_ca_cert.crt root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/.
+    # copy across the parameter files (we built them on the hub)
+    ssh $QUIET root@$KOJI_WEB_FQDN mkdir -p $SCRIPT_DIR
+    scp $QUIET $SCRIPT_DIR/koji-parameters.sh root@$KOJI_WEB_FQDN:$SCRIPT_DIR/koji-parameters.sh
+    # pull down the required scripts
+    ssh $QUIET root@$KOJI_WEB_FQDN "curl $SILENT $SCRIPT_GIT/koji-deploy-web.sh > $SCRIPT_DIR/koji-deploy-web.sh"
+    # make them executeable
+    ssh $QUIET root@$KOJI_WEB_FQDN "chmod o+x $SCRIPT_DIR/*"
+    
+    # connect to nfs share
+    ssh $QUIET root@$KOJI_WEB_FQDN $SCRIPT_DIR/koji-deploy-nfs-client.sh $DEBUG
+    # deploy koji builder
+    ssh $QUIET root@$KOJI_WEB_FQDN $SCRIPT_DIR/koji-deploy-web.sh $KOJI_WEB_FQDN $DEBUG
+fi
+
 # add builders
-echo "We will now deploy koji to your build servers"
+# if builders added as command line parameters, use those
-MSG="Press <enter> to use your hub ($KOJI_HUB_FQDN), or enter the FQDN of your first build server) "
+if [ $KOJI_BUILD_FQDN ] ; then
-BSNO=1
+    for each $FQDN in ${KOJI_HUB_FQDN} ; do
-while true ; do
+        deploy_builder($FQDN)
-    read -p "Build Server FQDN: " KOJI_BUILD_FQDN
+    done
+else
+# otherwise prompt for builders
+    echo "We will now deploy koji to your build servers"
+    MSG="Press <enter> to use your hub ($KOJI_HUB_FQDN), or enter the FQDN of your first build server) "
+    BSNO=1
+    while true ; do
+        read -p "Build Server FQDN: " FQDN
         MSG="Add another Build Server (will stop asking if left blank) "
-    if [ -z $KOJI_BUILD_FQDN ] ; then
+        if [ -z $FQDN ] ; then
-        if [[ ! $BSNO -gt 1 ]] ; then
+            # if blank the first time add builder to the hub
-            koji-add-builder.sh $DEBUG
+            if [[ $BSNO -gt 1 ]] ; then
-        fi
                 break
+            else
+                FQDN=$KOJI_HUB_FQDN
             fi
+        fi
+        deploy_builder($FQDN)
+        ((BSNO=BSNO+1))
+    done
+fi
+# 
+koji-bootstrap-build.sh
+
+exit 0
+
+deploy_builder () {
+    local FQDN=$1
     # check if server available on port 22
-    until (nc -z $KOJI_BUILD_FQDN 22)
+    until (nc -z $FQDN 22)
     do
-        echo "I cannot connect to $KOJI_BUILD_FQDN! Is it online? "
+        echo "I cannot connect to $FQDN! Is it online? "
         echo "Options:"
         echo "- FQDN=<FQDN of build server> # if you've mis-typed the name"
         echo "- IP=<IP of build server> # I'll add this to your /etc/hosts file"
@@ -152,11 +228,11 @@ while true ; do
             IP=* )
                 BIP=${RESPONSE#*=}
                 # add this builder into the hosts file
-                echo "$BIP  $KOJI_BUILD_FQDN\n" >> /etc/hosts
+                echo "$BIP  $FQDN\n" >> /etc/hosts
                 continue 
                 ;;
             FQDN=* )
-                KOJI_BUILD_FQDN=${RESPONSE#*=}
+                FQDN=${RESPONSE#*=}
                 continue
                 ;;
             *)
@@ -166,11 +242,5 @@ while true ; do
     done
 
     # we found the server and will deploy to it
-    echo "$KOJI_BUILD_FQDN will be Koji Build server No. $BSNO"
+    koji-add-builder.sh $FQDN $DEBUG
-    ((BSNO=BSNO+1))
+}
-    koji-add-builder.sh $KOJI_BUILD_FQDN $DEBUG
-
-done
-
-# 
-koji-bootstrap-build.sh

koji-setup/koji-add-builder.sh

@@ -63,16 +63,8 @@ fi
 sudo -u kojiadmin koji add-host "$KOJI_BUILD_FQDN" "$RPM_ARCH"
 # Add the host to the createrepo channel
 sudo -u kojiadmin koji add-host-to-channel "$KOJI_BUILD_FQDN" createrepo
-# A note on capacity
+# Add server capacity
 sudo -u kojiadmin koji edit-host --capacity="$KOJID_CAPACITY" "$KOJI_BUILD_FQDN"
-# Generate a certificate for the builder
-if [ ! -f $SCRIPT_DIR/koji-gencert.sh ] ; then
-    curl $SILENT $SCRIPT_GIT/koji-gencert.sh > $SCRIPT_DIR/koji-gencert.sh
-fi
-cp $SCRIPT_DIR/koji-gencert.sh $KOJI_PKI_DIR/.
-pushd "$KOJI_PKI_DIR"
-./koji-gencert.sh "$KOJI_BUILD_FQDN" "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/CN=$KOJI_BUILD_FQDN"
-popd
 # check if local install or remote
 if [[ $KOJI_BUILD_FQDN = $KOJI_HUB_FQDN ]] ; then
     # deploy locally

koji-setup/koji-deploy-hub.sh

@@ -168,10 +168,13 @@ sudo -u koji psql koji koji < /usr/share/doc/koji/docs/schema.sql
 # Authorize Koji-web and Koji-hub resources
 cat > "$POSTGRES_DIR"/data/pg_hba.conf <<- EOF
 #TYPE    DATABASE    USER    CIDR-ADDRESS    METHOD
-host     koji        all     127.0.0.1/32    trust
+local    koji        koji                    trust
-host     koji        all     ::1/128         trust
+local    all         postgres                peer
-local    koji        all                     trust
 EOF
+# possible alternatives
+#host     koji        koji    127.0.0.1/32    trust
+#host     koji        koji    ::1/128         trust
+
 systemctl reload postgresql
 
 # Bootstrapping the initial koji admin user into the PostgreSQL database
@@ -259,7 +262,7 @@ setsebool -P httpd_can_network_connect 1
 cat > /etc/koji.conf <<- EOF
 [koji]
 server = $KOJI_URL/kojihub
-weburl = $KOJI_URL/koji
+weburl = $KOJI_WEB_URL/koji
 topurl = $KOJI_URL/kojifiles
 topdir = $KOJI_DIR
 cert = ~/.koji/client.crt
@@ -272,7 +275,7 @@ EOF
 mkdir -p "$KOJI_DIR"/{packages,repos,work,scratch,repos-dist}
 chown -R "$HTTPD_USER":"$HTTPD_USER" "$KOJI_DIR"
 
-# twealk SELinux to allow $HTTPD_USER write access
+# tweak SELinux to allow $HTTPD_USER write access
 setsebool -P allow_httpd_anon_write=1
 semanage fcontext -a -t public_content_rw_t "$KOJI_DIR(/.*)?"
 restorecon -r -v $KOJI_DIR
@@ -335,11 +338,9 @@ firewall-cmd --reload
 # enable and start the httpd service
 systemctl enable --now httpd
 
-
 ## TEST KOJI CONNECTIVITY
 sudo -u kojiadmin koji moshimoshi
 
-
 ## KOJIRA - DNF|YUM REPOSITORY CREATION AND MAINTENANCE
 # Add the user entry for the kojira user
 sudo -u kojiadmin koji add-user kojira

koji-setup/koji-deploy-web.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+$KOJI_WEB_FQDN=$1
+DEBUG=
+SILENT="-s"
+QUIET="-q"
+for param in $2 ; do
+  if [ $param ] ; then
+    case $param in
+	    debug )
+		    DEBUG="debug" ;;
+    esac
+  else
+	  break
+  fi
+done
+
+if [ $DEBUG ] ; then
+    set -xe
+    SILENT=
+    QUIET="-v"
+fi
+
+# load required parameters
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+if [ ! -f "$SCRIPT_DIR"/koji-parameters.sh ] ; then
+    echo "$SCRIPT_DIR/koji-parameters.sh NOT found - aborting"
+    exit 1
+fi
+source "$SCRIPT_DIR"/koji-parameters.sh
+
+# Install all the required packages (some live in the epel repo, so we may need to install that too)
+if [[ -z $(dnf list installed | grep epel-release) ]] ; then
+    dnf config-manager --set-enabled powertools $QUIET
+	dnf install -y epel-release $QUIET
+fi
+dnf install -y mod_ssl koji-web $QUIET
+
+# install locally
+mkdir -p /etc/kojiweb
+cat > /etc/kojiweb/web.conf <<- EOF
+[web]
+SiteName = koji
+KojiHubURL = $KOJI_URL/kojihub
+KojiFilesURL = $KOJI_URL/kojifiles
+WebCert = $KOJI_PKI_DIR/kojiweb.pem
+ClientCA = $KOJI_PKI_DIR/koji_ca_cert.crt
+KojiHubCA = $KOJI_PKI_DIR/koji_ca_cert.crt
+LoginTimeout = 72
+Secret = NITRA_IS_NOT_CLEAR
+LibPath = /usr/share/koji-web/lib
+LiteralFooter = True
+EOF
+
+mkdir -p /etc/httpd/conf.d
+cat > /etc/httpd/conf.d/kojiweb.conf <<- EOF
+Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py"
+<Directory "/usr/share/koji-web/scripts">
+    Options ExecCGI
+    SetHandler wsgi-script
+    Require all granted
+</Directory>
+Alias /koji-static "/usr/share/koji-web/static"
+<Directory "/usr/share/koji-web/static">
+    Options None
+    AllowOverride None
+    Require all granted
+</Directory>
+EOF
+
+# SELinux changes to allow httpd network access
+setsebool -P httpd_can_network_connect 1
+
+## Apache Configuration Files
+mkdir -p /etc/httpd/conf.d
+cat > /etc/httpd/conf.d/ssl.conf <<- EOF
+ServerName $KOJI_WEB_FQDN
+
+Listen 443 https
+
+#SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog
+
+#SSLSessionCache         shmcb:/run/httpd/sslcache(512000)
+
+SSLRandomSeed startup file:/dev/urandom  256
+SSLRandomSeed connect builtin
+
+<VirtualHost _default_:443>
+    ErrorLog /var/log/httpd/ssl_error_log
+    TransferLog /var/log/httpd/ssl_access_log
+    LogLevel warn
+
+    SSLEngine on
+    SSLHonorCipherOrder on
+    SSLCipherSuite PROFILE=SYSTEM
+    SSLProxyCipherSuite PROFILE=SYSTEM
+
+    SSLCertificateFile $KOJI_PKI_DIR/kojihub.pem
+    SSLCertificateKeyFile $KOJI_PKI_DIR/private/kojihub.key
+    SSLCertificateChainFile $KOJI_PKI_DIR/koji_ca_cert.crt
+    SSLCACertificateFile $KOJI_PKI_DIR/koji_ca_cert.crt
+    SSLVerifyClient require
+    SSLVerifyDepth 10
+
+    <FilesMatch "\.(cgi|shtml|phtml|php3?)$">
+        SSLOptions +StdEnvVars
+    </FilesMatch>
+    <Directory "/var/www/cgi-bin">
+        SSLOptions +StdEnvVars
+    </Directory>
+
+    CustomLog /var/log/httpd/ssl_request_log "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b"
+</VirtualHost>
+EOF
+
+mkdir -p /etc/httpd/conf.modules.d
+cat > /etc/httpd/conf.modules.d/wsgi.conf <<- EOF
+WSGISocketPrefix /run/httpd/wsgi
+EOF
+cat > /etc/httpd/conf.modules.d/ssl.conf <<- EOF
+LoadModule ssl_module lib/httpd/modules/mod_ssl.so
+EOF
+
+# allow httpd access through firewall
+firewall-cmd --permanent --add-service=http
+firewall-cmd --permanent --add-service=https
+firewall-cmd --reload
+
+# enable and start the httpd service
+systemctl enable --now httpd