From 73faa723bf24fa150fb46abc664582f329194ec6 Mon Sep 17 00:00:00 2001 From: Trevor Batley Date: Mon, 11 Sep 2023 13:02:10 +1000 Subject: [PATCH] first cut web deploy --- install-koji-farm.sh | 128 ++++++++++++++++++++++++-------- koji-setup/koji-add-builder.sh | 10 +-- koji-setup/koji-deploy-hub.sh | 15 ++-- koji-setup/koji-deploy-web.sh | 132 +++++++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 45 deletions(-) create mode 100644 koji-setup/koji-deploy-web.sh diff --git a/install-koji-farm.sh b/install-koji-farm.sh index 2f101de..6a88f16 100644 --- a/install-koji-farm.sh +++ b/install-koji-farm.sh @@ -2,11 +2,18 @@ set -e DEBUG= -for param in $1 $2 $3 ; do +KOJI_HUB_FQDN="$(hostname -f)" +KOJI_WEB_FQDN=$KOJI_HUB_FQDN +KOJI_Build_FQDN= +for param in $1 $2 $3 $4 $5 $6 $7; do if [ $param ] ; then case $param in - debug ) - DEBUG="debug" ;; + debug ) + DEBUG="debug" ;; + web=* ) + KOJI_WEB_FQDN=${param#=*} ;; + build=* ) + KOJI_BUILD_FQDN=$KOJI_BUILD_FQDN" "${param#=*} ;; esac else break @@ -27,6 +34,7 @@ SCRIPT_DIR="$(echo ~)/bin" mkdir -p $SCRIPT_DIR curl $SILENT $SCRIPT_GIT/koji-deploy-hub.sh > $SCRIPT_DIR/koji-deploy-hub.sh +curl $SILENT $SCRIPT_GIT/koji-deploy-web.sh > $SCRIPT_DIR/koji-deploy-web.sh curl $SILENT $SCRIPT_GIT/koji-add-builder.sh > $SCRIPT_DIR/koji-add-builder.sh curl $SILENT $SCRIPT_GIT/koji-bootstrap-build.sh > $SCRIPT_DIR/koji-bootstrap-build.sh curl $SILENT $SCRIPT_GIT/koji-add-user.sh > $SCRIPT_DIR/koji-add-user.sh @@ -68,11 +76,10 @@ while true ; do done # setup default parameters -KOJI_HUB_FQDN="$(hostname -f)" HTTPD_USER=apache HTTPD_DOCUMENT_ROOT=/var/www/html KOJI_DIR="/mnt/koji" -TAG_NAME="smeos" +TAG_NAME="smeos11,amecontribs11" KOJI_PKI_DIR=/etc/pki/koji RPM_ARCH="x86_64 noarch" KOJID_CAPACITY=16 @@ -119,23 +126,92 @@ chmod o+x $SCRIPT_DIR/koji-parameters.sh # deploy the central koji components koji-deploy-hub.sh $DEBUG -# add builders -echo "We will now deploy koji to your build servers" -MSG="Press to use your hub ($KOJI_HUB_FQDN), or enter the FQDN of your first build server) " -BSNO=1 -while true ; do - read -p "Build Server FQDN: " KOJI_BUILD_FQDN - MSG="Add another Build Server (will stop asking if left blank) " - if [ -z $KOJI_BUILD_FQDN ] ; then - if [[ ! $BSNO -gt 1 ]] ; then - koji-add-builder.sh $DEBUG - fi - break +# deploy the web koji components +if [[ $KOJI_WEB_FQDN = $KOJI_HUB_FQDN ]] ; then + # deploy locally (default) + koji_deploy_web.sh $KOJI_WEB_FQDN $DEBUG +else + # deploy remotely to $KOJI_WEB_FQDN + # check that I can conmnect + if [ ! nc -z $KOJI_WEB_FQDN 22 2>/dev/null ] ; then + echo "I cannot connect to $KOJI_WEB_FQDN! Is it online? " + echo "Options:" + echo "- turn on the server" + echo "- add this server into the /etc/hosts file on this server" + exit 1 + fi + # generate a hub ssh key if there isn't one already (for scp & ssh to web server) + if [ ! -f /root/.ssh/id-rsa ] ; then + # create a ssh key on build server + mkdir -p ~/.ssh + ssh-keygen -t rsa -f /root/.ssh/id_rsa -N "" fi + # copy the server key into authorized keys on the web server + ssh-copy-id -i ~/.ssh/id_rsa.pub $KOJI_WEB_FQDN + # find the IP of the web server + WEB_IP=$(ssh root@$KOJI_WEB_FQDN "hostname -I") + # add web server into allowed access to db + cat >> "$POSTGRES_DIR"/data/pg_hba.conf <<- EOF +host koji koji $WEB_IP/32 scram-sha-256 +EOF + systemctl reload postgresql + # copy across the ssl keys + ssh $QUIET root@$KOJI_WEB_FQDN mkdir -p $KOJI_PKI_DIR/private + scp $QUIET $KOJI_PKI_DIR/$KOJI_WEB_FQDN.pem root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/. + scp $QUIET $KOJI_PKI_DIR/kojihub.pem root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/. + scp $QUIET $KOJI_PKI_DIR/private/kojihub.key root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/private/. + scp $QUIET $KOJI_PKI_DIR/koji_ca_cert.crt root@$KOJI_WEB_FQDN:$KOJI_PKI_DIR/. + # copy across the parameter files (we built them on the hub) + ssh $QUIET root@$KOJI_WEB_FQDN mkdir -p $SCRIPT_DIR + scp $QUIET $SCRIPT_DIR/koji-parameters.sh root@$KOJI_WEB_FQDN:$SCRIPT_DIR/koji-parameters.sh + # pull down the required scripts + ssh $QUIET root@$KOJI_WEB_FQDN "curl $SILENT $SCRIPT_GIT/koji-deploy-web.sh > $SCRIPT_DIR/koji-deploy-web.sh" + # make them executeable + ssh $QUIET root@$KOJI_WEB_FQDN "chmod o+x $SCRIPT_DIR/*" + + # connect to nfs share + ssh $QUIET root@$KOJI_WEB_FQDN $SCRIPT_DIR/koji-deploy-nfs-client.sh $DEBUG + # deploy koji builder + ssh $QUIET root@$KOJI_WEB_FQDN $SCRIPT_DIR/koji-deploy-web.sh $KOJI_WEB_FQDN $DEBUG +fi + +# add builders +# if builders added as command line parameters, use those +if [ $KOJI_BUILD_FQDN ] ; then + for each $FQDN in ${KOJI_HUB_FQDN} ; do + deploy_builder($FQDN) + done +else +# otherwise prompt for builders + echo "We will now deploy koji to your build servers" + MSG="Press to use your hub ($KOJI_HUB_FQDN), or enter the FQDN of your first build server) " + BSNO=1 + while true ; do + read -p "Build Server FQDN: " FQDN + MSG="Add another Build Server (will stop asking if left blank) " + if [ -z $FQDN ] ; then + # if blank the first time add builder to the hub + if [[ $BSNO -gt 1 ]] ; then + break + else + FQDN=$KOJI_HUB_FQDN + fi + fi + deploy_builder($FQDN) + ((BSNO=BSNO+1)) + done +fi +# +koji-bootstrap-build.sh + +exit 0 + +deploy_builder () { + local FQDN=$1 # check if server available on port 22 - until (nc -z $KOJI_BUILD_FQDN 22) + until (nc -z $FQDN 22) do - echo "I cannot connect to $KOJI_BUILD_FQDN! Is it online? " + echo "I cannot connect to $FQDN! Is it online? " echo "Options:" echo "- FQDN= # if you've mis-typed the name" echo "- IP= # I'll add this to your /etc/hosts file" @@ -152,11 +228,11 @@ while true ; do IP=* ) BIP=${RESPONSE#*=} # add this builder into the hosts file - echo "$BIP $KOJI_BUILD_FQDN\n" >> /etc/hosts + echo "$BIP $FQDN\n" >> /etc/hosts continue ;; FQDN=* ) - KOJI_BUILD_FQDN=${RESPONSE#*=} + FQDN=${RESPONSE#*=} continue ;; *) @@ -166,11 +242,5 @@ while true ; do done # we found the server and will deploy to it - echo "$KOJI_BUILD_FQDN will be Koji Build server No. $BSNO" - ((BSNO=BSNO+1)) - koji-add-builder.sh $KOJI_BUILD_FQDN $DEBUG - -done - -# -koji-bootstrap-build.sh \ No newline at end of file + koji-add-builder.sh $FQDN $DEBUG +} diff --git a/koji-setup/koji-add-builder.sh b/koji-setup/koji-add-builder.sh index 9ab1bca..dfab2c8 100644 --- a/koji-setup/koji-add-builder.sh +++ b/koji-setup/koji-add-builder.sh @@ -63,16 +63,8 @@ fi sudo -u kojiadmin koji add-host "$KOJI_BUILD_FQDN" "$RPM_ARCH" # Add the host to the createrepo channel sudo -u kojiadmin koji add-host-to-channel "$KOJI_BUILD_FQDN" createrepo -# A note on capacity +# Add server capacity sudo -u kojiadmin koji edit-host --capacity="$KOJID_CAPACITY" "$KOJI_BUILD_FQDN" -# Generate a certificate for the builder -if [ ! -f $SCRIPT_DIR/koji-gencert.sh ] ; then - curl $SILENT $SCRIPT_GIT/koji-gencert.sh > $SCRIPT_DIR/koji-gencert.sh -fi -cp $SCRIPT_DIR/koji-gencert.sh $KOJI_PKI_DIR/. -pushd "$KOJI_PKI_DIR" -./koji-gencert.sh "$KOJI_BUILD_FQDN" "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/CN=$KOJI_BUILD_FQDN" -popd # check if local install or remote if [[ $KOJI_BUILD_FQDN = $KOJI_HUB_FQDN ]] ; then # deploy locally diff --git a/koji-setup/koji-deploy-hub.sh b/koji-setup/koji-deploy-hub.sh index 1c4281c..3aef2b7 100644 --- a/koji-setup/koji-deploy-hub.sh +++ b/koji-setup/koji-deploy-hub.sh @@ -168,10 +168,13 @@ sudo -u koji psql koji koji < /usr/share/doc/koji/docs/schema.sql # Authorize Koji-web and Koji-hub resources cat > "$POSTGRES_DIR"/data/pg_hba.conf <<- EOF #TYPE DATABASE USER CIDR-ADDRESS METHOD -host koji all 127.0.0.1/32 trust -host koji all ::1/128 trust -local koji all trust +local koji koji trust +local all postgres peer EOF +# possible alternatives +#host koji koji 127.0.0.1/32 trust +#host koji koji ::1/128 trust + systemctl reload postgresql # Bootstrapping the initial koji admin user into the PostgreSQL database @@ -259,7 +262,7 @@ setsebool -P httpd_can_network_connect 1 cat > /etc/koji.conf <<- EOF [koji] server = $KOJI_URL/kojihub -weburl = $KOJI_URL/koji +weburl = $KOJI_WEB_URL/koji topurl = $KOJI_URL/kojifiles topdir = $KOJI_DIR cert = ~/.koji/client.crt @@ -272,7 +275,7 @@ EOF mkdir -p "$KOJI_DIR"/{packages,repos,work,scratch,repos-dist} chown -R "$HTTPD_USER":"$HTTPD_USER" "$KOJI_DIR" -# twealk SELinux to allow $HTTPD_USER write access +# tweak SELinux to allow $HTTPD_USER write access setsebool -P allow_httpd_anon_write=1 semanage fcontext -a -t public_content_rw_t "$KOJI_DIR(/.*)?" restorecon -r -v $KOJI_DIR @@ -335,11 +338,9 @@ firewall-cmd --reload # enable and start the httpd service systemctl enable --now httpd - ## TEST KOJI CONNECTIVITY sudo -u kojiadmin koji moshimoshi - ## KOJIRA - DNF|YUM REPOSITORY CREATION AND MAINTENANCE # Add the user entry for the kojira user sudo -u kojiadmin koji add-user kojira diff --git a/koji-setup/koji-deploy-web.sh b/koji-setup/koji-deploy-web.sh new file mode 100644 index 0000000..37c37ea --- /dev/null +++ b/koji-setup/koji-deploy-web.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 + +set -e +$KOJI_WEB_FQDN=$1 +DEBUG= +SILENT="-s" +QUIET="-q" +for param in $2 ; do + if [ $param ] ; then + case $param in + debug ) + DEBUG="debug" ;; + esac + else + break + fi +done + +if [ $DEBUG ] ; then + set -xe + SILENT= + QUIET="-v" +fi + +# load required parameters +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +if [ ! -f "$SCRIPT_DIR"/koji-parameters.sh ] ; then + echo "$SCRIPT_DIR/koji-parameters.sh NOT found - aborting" + exit 1 +fi +source "$SCRIPT_DIR"/koji-parameters.sh + +# Install all the required packages (some live in the epel repo, so we may need to install that too) +if [[ -z $(dnf list installed | grep epel-release) ]] ; then + dnf config-manager --set-enabled powertools $QUIET + dnf install -y epel-release $QUIET +fi +dnf install -y mod_ssl koji-web $QUIET + +# install locally +mkdir -p /etc/kojiweb +cat > /etc/kojiweb/web.conf <<- EOF +[web] +SiteName = koji +KojiHubURL = $KOJI_URL/kojihub +KojiFilesURL = $KOJI_URL/kojifiles +WebCert = $KOJI_PKI_DIR/kojiweb.pem +ClientCA = $KOJI_PKI_DIR/koji_ca_cert.crt +KojiHubCA = $KOJI_PKI_DIR/koji_ca_cert.crt +LoginTimeout = 72 +Secret = NITRA_IS_NOT_CLEAR +LibPath = /usr/share/koji-web/lib +LiteralFooter = True +EOF + +mkdir -p /etc/httpd/conf.d +cat > /etc/httpd/conf.d/kojiweb.conf <<- EOF +Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py" + + Options ExecCGI + SetHandler wsgi-script + Require all granted + +Alias /koji-static "/usr/share/koji-web/static" + + Options None + AllowOverride None + Require all granted + +EOF + +# SELinux changes to allow httpd network access +setsebool -P httpd_can_network_connect 1 + +## Apache Configuration Files +mkdir -p /etc/httpd/conf.d +cat > /etc/httpd/conf.d/ssl.conf <<- EOF +ServerName $KOJI_WEB_FQDN + +Listen 443 https + +#SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog + +#SSLSessionCache shmcb:/run/httpd/sslcache(512000) + +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin + + + ErrorLog /var/log/httpd/ssl_error_log + TransferLog /var/log/httpd/ssl_access_log + LogLevel warn + + SSLEngine on + SSLHonorCipherOrder on + SSLCipherSuite PROFILE=SYSTEM + SSLProxyCipherSuite PROFILE=SYSTEM + + SSLCertificateFile $KOJI_PKI_DIR/kojihub.pem + SSLCertificateKeyFile $KOJI_PKI_DIR/private/kojihub.key + SSLCertificateChainFile $KOJI_PKI_DIR/koji_ca_cert.crt + SSLCACertificateFile $KOJI_PKI_DIR/koji_ca_cert.crt + SSLVerifyClient require + SSLVerifyDepth 10 + + + SSLOptions +StdEnvVars + + + SSLOptions +StdEnvVars + + + CustomLog /var/log/httpd/ssl_request_log "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" + +EOF + +mkdir -p /etc/httpd/conf.modules.d +cat > /etc/httpd/conf.modules.d/wsgi.conf <<- EOF +WSGISocketPrefix /run/httpd/wsgi +EOF +cat > /etc/httpd/conf.modules.d/ssl.conf <<- EOF +LoadModule ssl_module lib/httpd/modules/mod_ssl.so +EOF + +# allow httpd access through firewall +firewall-cmd --permanent --add-service=http +firewall-cmd --permanent --add-service=https +firewall-cmd --reload + +# enable and start the httpd service +systemctl enable --now httpd