From b317fffb3fccff4e560130a4ab87186286118e79 Mon Sep 17 00:00:00 2001 From: Trevor Batley Date: Mon, 28 Aug 2023 14:44:38 +1000 Subject: [PATCH] Initial load --- .DS_Store | Bin 0 -> 6148 bytes README.md | 56 ++++- inv/hosts.example | 6 + koji-setup/bootstrap-build.sh | 53 ++++ koji-setup/deploy-git.sh | 167 +++++++++++++ koji-setup/deploy-koji-builder.sh | 59 +++++ koji-setup/deploy-koji-nfs-client.sh | 28 +++ koji-setup/deploy-koji-nfs-server.sh | 18 ++ koji-setup/deploy-koji.sh | 355 +++++++++++++++++++++++++++ koji-setup/deploy-mash.sh | 41 ++++ koji-setup/deploy-upstreams.sh | 16 ++ koji-setup/gencert.sh | 17 ++ koji-setup/globals.sh | 28 +++ koji-setup/mash.sh | 116 +++++++++ koji-setup/parameters.sh | 41 ++++ koji.yaml | 38 +++ 16 files changed, 1037 insertions(+), 2 deletions(-) create mode 100644 .DS_Store create mode 100644 inv/hosts.example create mode 100755 koji-setup/bootstrap-build.sh create mode 100755 koji-setup/deploy-git.sh create mode 100755 koji-setup/deploy-koji-builder.sh create mode 100755 koji-setup/deploy-koji-nfs-client.sh create mode 100755 koji-setup/deploy-koji-nfs-server.sh create mode 100755 koji-setup/deploy-koji.sh create mode 100755 koji-setup/deploy-mash.sh create mode 100755 koji-setup/deploy-upstreams.sh create mode 100755 koji-setup/gencert.sh create mode 100644 koji-setup/globals.sh create mode 100755 koji-setup/mash.sh create mode 100644 koji-setup/parameters.sh create mode 100644 koji.yaml diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..b15f45525ed9a7a657ee1dde4242ad9f46927223 GIT binary patch literal 6148 zcmeHKJ5Iw;5S)b+k!T_+%69<@ZeT^>1Y7`-Pymq};iPxPxi}iLpT;sYk!Ye>Y2UoJ zZ)Yt};q?Nr^=bD2tN<+Oj`;FrZhr4Rv#W|2k98(Qs}zs|Qw6T`xbpu0NdII0pOSQv0#e|j6tKnSX|v`lRd1cWocG#Bf2MoRH{Ff< opm2$HOpJESgSX?mNXoqCd){}2V`9)54?0ml1Fnlq3jDVMKb7MckpKVy literal 0 HcmV?d00001 diff --git a/README.md b/README.md index e278348..f0a5088 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,55 @@ -# smeserver-koji +# smeserver-koji -Scripts to build/maintain koji infrastructure \ No newline at end of file +# Koji Setup Scripts + +The purpose of these scripts it to enable setting up a koji environment quickly +with reasonable configurations. + +## Assumptions + +* All scripts are run as the root user +* The root user has a password set +* Basic configurations (e.g. network, time, etc.) have been applied +* Only one koji builder is required + +## Unsupported Environments + +* Systems that are not starting as dedicated and clean +* Systems that are not based on Clear Linux OS* + +For unsupported environments, it will be up to the sysadmin to proceed at their +own discretion and fix issues that may arise on their own. + +## Getting Going + +1. Edit parameters.sh as needed. If running in a production environment, be +sure to supply reasonable SSL certificate field values. + +1. Run the required following scripts + + deploy-koji.sh + bootstrp-build.sh + +1. Optionally, for supporting a full DevOps workflow, also run + + deploy-mash.sh + deploy-git.sh + deploy-upstreams.sh + +If koji builder machine is not the same as koji master machine: + +1. On the koji master machine, run + + deploy-koji-nfs-server.sh + +1. Copy the koji builder certificate from the koji master machine to the koji +builder machine + + scp "$KOJI_PKI_DIR/$KOJI_SLAVE_FQDN.pem" "$KOJI_SLAVE_FQDN":"$KOJI_PKI_DIR" + +1. On the koji builder machine, run + + deploy-koji-nfs-client.sh + deploy-koji-builder.sh + +*Other names and brands may be claimed as the property of others. diff --git a/inv/hosts.example b/inv/hosts.example new file mode 100644 index 0000000..369005c --- /dev/null +++ b/inv/hosts.example @@ -0,0 +1,6 @@ +all: + hosts: + koji-all-in-one: + ansible_port: 22 + ansible_host: localhost + ansible_user: test diff --git a/koji-setup/bootstrap-build.sh b/koji-setup/bootstrap-build.sh new file mode 100755 index 0000000..f52484a --- /dev/null +++ b/koji-setup/bootstrap-build.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/globals.sh +source "$SCRIPT_DIR"/parameters.sh + +STAGING_RPM_DIR="$KOJI_DIR/work/imported-rpms" +STAGING_RPM_SRC_DIR="$STAGING_RPM_DIR/src" +STAGING_RPM_BIN_DIR="$STAGING_RPM_DIR/bin" +STAGING_RPM_DEBUG_DIR="$STAGING_RPM_DIR/debug" + +import_koji_pkg() { + local src_dir="$1" + local dst_dir="$2" + local search_pattern="$3" + cp -r "$src_dir" "$dst_dir" + chown -R "$HTTPD_USER":"$HTTPD_USER" "$dst_dir" + find "$dst_dir" -name "$search_pattern" -exec koji import --link {} + > /dev/null +} + +if [[ -n "$SRC_RPM_DIR" && -n "$BIN_RPM_DIR" ]]; then + ADMIN_KOJI_DIR="$(echo ~kojiadmin)/.koji" + cp -r "$ADMIN_KOJI_DIR" "$HOME/.koji" + mkdir -p "$STAGING_RPM_DIR" + chown -R "$HTTPD_USER":"$HTTPD_USER" "$STAGING_RPM_DIR" + + import_koji_pkg "$SRC_RPM_DIR" "$STAGING_RPM_SRC_DIR" "*.src.rpm" + import_koji_pkg "$BIN_RPM_DIR" "$STAGING_RPM_BIN_DIR" "*.$RPM_ARCH.rpm" + if [[ -n "$DEBUG_RPM_DIR" ]]; then + import_koji_pkg "$DEBUG_RPM_DIR" "$STAGING_RPM_DEBUG_DIR" "*.$RPM_ARCH.rpm" + fi + + rm -rf "$STAGING_RPM_DIR" "$HOME/.koji" +fi +sudo -u kojiadmin koji add-tag dist-"$TAG_NAME" +sudo -u kojiadmin koji edit-tag dist-"$TAG_NAME" -x mock.package_manager=dnf +if [[ -n "$SRC_RPM_DIR" && -n "$BIN_RPM_DIR" ]]; then + sudo -u kojiadmin koji list-pkgs --quiet | xargs sudo -u kojiadmin koji add-pkg --owner kojiadmin dist-"$TAG_NAME" + sudo -u kojiadmin koji list-untagged | xargs -n 1 -P 100 sudo -u kojiadmin koji call tagBuildBypass dist-"$TAG_NAME" > /dev/null +fi +sudo -u kojiadmin koji add-tag --parent dist-"$TAG_NAME" --arches "$RPM_ARCH" dist-"$TAG_NAME"-build +sudo -u kojiadmin koji add-target dist-"$TAG_NAME" dist-"$TAG_NAME"-build +sudo -u kojiadmin koji add-group dist-"$TAG_NAME"-build build +sudo -u kojiadmin koji add-group dist-"$TAG_NAME"-build srpm-build +sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build build autoconf automake automake-dev binutils bzip2 clr-rpm-config coreutils cpio diffutils elfutils file gawk gcc gcc-dev gettext gettext-bin git glibc-dev glibc-locale glibc-utils grep gzip hostname libc6-dev libcap libtool libtool-dev linux-libc-headers m4 make netbase nss-altfiles patch pigz pkg-config pkg-config-dev rpm sed shadow systemd-lib tar unzip which xz +sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build srpm-build coreutils cpio curl-bin elfutils file git glibc-utils grep gzip make pigz plzip rpm sed shadow tar unzip wget xz +if [[ -n "$EXTERNAL_REPO" ]]; then + sudo -u kojiadmin koji add-external-repo -t dist-"$TAG_NAME"-build dist-"$TAG_NAME"-external-repo "$EXTERNAL_REPO" +fi +sudo -u kojiadmin koji regen-repo dist-"$TAG_NAME"-build diff --git a/koji-setup/deploy-git.sh b/koji-setup/deploy-git.sh new file mode 100755 index 0000000..55eaff0 --- /dev/null +++ b/koji-setup/deploy-git.sh @@ -0,0 +1,167 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/globals.sh +source "$SCRIPT_DIR"/parameters.sh + +swupd bundle-add scm-server || : +check_dependency gitolite +check_dependency git + +## GITOLITE SETUP +mkdir -p "$GIT_DIR" +chown -R "$GIT_USER":"$GIT_USER" "$GIT_DIR" +# Add symlink for backwards compatibility +if [[ "$GIT_DIR" != "$GIT_DEFAULT_DIR" ]]; then + if [ "$(ls -A "$GIT_DEFAULT_DIR")" ]; then + mv "$GIT_DEFAULT_DIR" "$GIT_DEFAULT_DIR".old + else + rm -rf "$GIT_DEFAULT_DIR" + fi + ln -sf "$GIT_DIR" "$GIT_DEFAULT_DIR" + chown -h "$GIT_USER":"$GIT_USER" "$GIT_DEFAULT_DIR" +fi +GITOLITE_PUB_KEY_FILE="$GIT_DEFAULT_DIR/gitolite.pub" +echo "$GITOLITE_PUB_KEY" > "$GITOLITE_PUB_KEY_FILE" +chown "$GIT_USER":"$GIT_USER" "$GITOLITE_PUB_KEY_FILE" +sudo -u "$GIT_USER" gitolite setup -pk "$GITOLITE_PUB_KEY_FILE" +usermod -s /bin/bash gitolite + +if $IS_ANONYMOUS_GIT_NEEDED; then + swupd bundle-add httpd || : + check_dependency httpd + + ## GIT PROTOCOL CLONING + mkdir -p /etc/systemd/system + cat > /etc/systemd/system/git-daemon.service <<- EOF + [Unit] + Description=Git Daemon + + [Service] + ExecStart=/usr/bin/git daemon --export-all --reuseaddr --base-path=$GIT_DEFAULT_DIR/repositories $GIT_DEFAULT_DIR/repositories + + Restart=always + RestartSec=500ms + + User=$GIT_USER + Group=$GIT_USER + + [Install] + WantedBy=multi-user.target + EOF + systemctl daemon-reload + systemctl enable --now git-daemon + + + ## CGIT WEB INTERFACE + cat > /etc/cgitrc <<- EOF + # Enable caching of up to 1000 output entries + cache-size=10 + + # Specify the css url + css=/cgit-data/cgit.css + + # Show extra links for each repository on the index page + enable-index-links=1 + + # Enable ASCII art commit history graph on the log pages + enable-commit-graph=1 + + # Show number of affected files per commit on the log pages + enable-log-filecount=1 + + # Show number of added/removed lines per commit on the log pages + enable-log-linecount=1 + + # Use a custom logo + logo=/cgit-data/cgit.png + + # Enable statistics per week, month and quarter + max-stats=quarter + + # Allow download of tar.gz, tar.bz2, and tar.xz formats + snapshots=tar.gz tar.bz2 tar.xz + + ## + ## List of common mimetypes + ## + mimetype.gif=image/gif + mimetype.html=text/html + mimetype.jpg=image/jpeg + mimetype.jpeg=image/jpeg + mimetype.pdf=application/pdf + mimetype.png=image/png + mimetype.svg=image/svg+xml + + # Enable syntax highlighting and about formatting + source-filter=/usr/libexec/cgit/filters/syntax-highlighting.py + about-filter=/usr/libexec/cgit/filters/about-formatting.sh + + ## + ## List of common readmes + ## + readme=:README.md + readme=:readme.md + readme=:README.mkd + readme=:readme.mkd + readme=:README.rst + readme=:readme.rst + readme=:README.html + readme=:readme.html + readme=:README.htm + readme=:readme.htm + readme=:README.txt + readme=:readme.txt + readme=:README + readme=:readme + readme=:INSTALL.md + readme=:install.md + readme=:INSTALL.mkd + readme=:install.mkd + readme=:INSTALL.rst + readme=:install.rst + readme=:INSTALL.html + readme=:install.html + readme=:INSTALL.htm + readme=:install.htm + readme=:INSTALL.txt + readme=:install.txt + readme=:INSTALL + readme=:install + + # Direct cgit to repository location managed by gitolite + remove-suffix=1 + project-list=$GIT_DEFAULT_DIR/projects.list + scan-path=$GIT_DEFAULT_DIR/repositories + EOF + + mkdir -p /etc/httpd/conf.modules.d + cat > /etc/httpd/conf.modules.d/cgid.conf <<- EOF + LoadModule cgid_module lib/httpd/modules/mod_cgid.so + ScriptSock /run/httpd/cgid.sock + EOF + + mkdir -p /etc/httpd/conf.d + cat > /etc/httpd/conf.d/cgit.conf <<- EOF + Alias /cgit-data /usr/share/cgit + + AllowOverride None + Options None + Require all granted + + + ScriptAlias /cgit /usr/libexec/cgit/cgi-bin/cgit + + AllowOverride None + Options ExecCGI + Require all granted + + EOF + usermod -a -G "$GIT_USER" "$HTTPD_USER" + + systemctl restart httpd + systemctl enable httpd +fi diff --git a/koji-setup/deploy-koji-builder.sh b/koji-setup/deploy-koji-builder.sh new file mode 100755 index 0000000..3883d91 --- /dev/null +++ b/koji-setup/deploy-koji-builder.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/globals.sh +source "$SCRIPT_DIR"/parameters.sh + +swupd bundle-add koji || : +check_dependency kojid + +# Create mock folders and permissions +mkdir -p /etc/mock/koji +mkdir -p /var/lib/mock +chown -R root:mock /var/lib/mock + +# Setup User Accounts +useradd -r kojibuilder +usermod -G mock kojibuilder + +# Kojid Configuration Files +if [[ "$KOJI_SLAVE_FQDN" = "$KOJI_MASTER_FQDN" ]]; then + KOJI_TOP_DIR="$KOJI_DIR" +else + KOJI_TOP_DIR="$KOJI_MOUNT_DIR" +fi +mkdir -p /etc/kojid +cat > /etc/kojid/kojid.conf <<- EOF +[kojid] +sleeptime=5 +maxjobs=16 +topdir=$KOJI_TOP_DIR +workdir=/tmp/koji +mockdir=/var/lib/mock +mockuser=kojibuilder +mockhost=generic-linux-gnu +user=$KOJI_SLAVE_FQDN +server=$KOJI_URL/kojihub +topurl=$KOJI_URL/kojifiles +use_createrepo_c=True +allowed_scms=$GIT_FQDN:/packages/* +cert = $KOJI_PKI_DIR/$KOJI_SLAVE_FQDN.pem +serverca = $KOJI_PKI_DIR/koji_ca_cert.crt +EOF + +if env | grep -q proxy; then + echo "yum_proxy = $https_proxy" >> /etc/kojid/kojid.conf + mkdir -p /etc/systemd/system/kojid.service.d + cat > /etc/systemd/system/kojid.service.d/00-proxy.conf <<- EOF + [Service] + Environment="http_proxy=$http_proxy" + Environment="https_proxy=$https_proxy" + Environment="no_proxy=$no_proxy" + EOF + systemctl daemon-reload +fi + +systemctl enable --now kojid diff --git a/koji-setup/deploy-koji-nfs-client.sh b/koji-setup/deploy-koji-nfs-client.sh new file mode 100755 index 0000000..2274166 --- /dev/null +++ b/koji-setup/deploy-koji-nfs-client.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/globals.sh +source "$SCRIPT_DIR"/parameters.sh + +KOJI_MOUNT_SERVICE="${KOJI_MOUNT_DIR:1}" +KOJI_MOUNT_SERVICE="${KOJI_MOUNT_SERVICE/\//-}".mount +mkdir -p /etc/systemd/system +cat > /etc/systemd/system/"$KOJI_MOUNT_SERVICE" <<- EOF +[Unit] +Description=Koji NFS Mount +After=network.target + +[Mount] +What=$KOJI_MASTER_FQDN:$KOJI_DIR +Where=$KOJI_MOUNT_DIR +Type=nfs +Options=defaults,ro + +[Install] +WantedBy=multi-user.target +EOF +systemctl daemon-reload +systemctl enable --now "$KOJI_MOUNT_SERVICE" diff --git a/koji-setup/deploy-koji-nfs-server.sh b/koji-setup/deploy-koji-nfs-server.sh new file mode 100755 index 0000000..25585e3 --- /dev/null +++ b/koji-setup/deploy-koji-nfs-server.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/globals.sh +source "$SCRIPT_DIR"/parameters.sh + +swupd bundle-add nfs-utils || : +check_dependency rpcbind +check_dependency rpc.nfsd + +# Export server directory to be mounted by clients +echo "$KOJI_DIR $KOJI_SLAVE_FQDN(ro,no_root_squash)" >> /etc/exports + +systemctl enable --now rpcbind +systemctl enable --now nfs-server diff --git a/koji-setup/deploy-koji.sh b/koji-setup/deploy-koji.sh new file mode 100755 index 0000000..d22b425 --- /dev/null +++ b/koji-setup/deploy-koji.sh @@ -0,0 +1,355 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/globals.sh +source "$SCRIPT_DIR"/parameters.sh + +swupd bundle-add koji || : +check_dependency koji +check_dependency httpd +check_dependency kojira +check_dependency postgres + +## SETTING UP SSL CERTIFICATES FOR AUTHENTICATION +mkdir -p "$KOJI_PKI_DIR"/{certs,private} +RANDFILE="$KOJI_PKI_DIR"/.rand +dd if=/dev/urandom of="$RANDFILE" bs=256 count=1 + +# Certificate generation +cat > "$KOJI_PKI_DIR"/ssl.cnf <<- EOF +HOME = $KOJI_PKI_DIR +RANDFILE = $RANDFILE + +[ca] +default_ca = ca_default + +[ca_default] +dir = $KOJI_PKI_DIR +certs = \$dir/certs +crl_dir = \$dir/crl +database = \$dir/index.txt +new_certs_dir = \$dir/newcerts +certificate = \$dir/%s_ca_cert.pem +private_key = \$dir/private/%s_ca_key.pem +serial = \$dir/serial +crl = \$dir/crl.pem +x509_extensions = usr_cert +name_opt = ca_default +cert_opt = ca_default +default_days = 3650 +default_crl_days = 30 +default_md = sha512 +preserve = no +policy = policy_match + +[policy_match] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[req] +default_bits = 4096 +default_keyfile = privkey.pem +default_md = sha512 +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca # The extensions to add to the self signed cert +string_mask = MASK:0x2002 + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_min = 2 +countryName_max = 2 +stateOrProvinceName = State or Province Name (full name) +localityName = Locality Name (eg, city) +0.organizationName = Organization Name (eg, company) +organizationalUnitName = Organizational Unit Name (eg, section) +commonName = Common Name (eg, your name or your server\'s hostname) +commonName_max = 64 +emailAddress = Email Address +emailAddress_max = 64 + +[req_attributes] +challengePassword = A challenge password +challengePassword_min = 8 +challengePassword_max = 64 +unstructuredName = An optional company name + +[usr_cert] +basicConstraints = CA:FALSE +nsComment = "OpenSSL Generated Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always + +[v3_ca] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer:always +basicConstraints = CA:true +EOF + +# Generate and trust CA +touch "$KOJI_PKI_DIR"/index.txt +echo 01 > "$KOJI_PKI_DIR"/serial +openssl genrsa -out "$KOJI_PKI_DIR"/private/koji_ca_cert.key 2048 +openssl req -subj "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=koji_ca/CN=$KOJI_MASTER_FQDN" -config "$KOJI_PKI_DIR"/ssl.cnf -new -x509 -days 3650 -key "$KOJI_PKI_DIR"/private/koji_ca_cert.key -out "$KOJI_PKI_DIR"/koji_ca_cert.crt -extensions v3_ca +mkdir -p /etc/ca-certs/trusted +cp -a "$KOJI_PKI_DIR"/koji_ca_cert.crt /etc/ca-certs/trusted +while true; do + if clrtrust generate; then + break + fi +done + +# Generate the koji component certificates and the admin certificate and generate a PKCS12 user certificate (for web browser) +cp "$SCRIPT_DIR"/gencert.sh "$KOJI_PKI_DIR" +pushd "$KOJI_PKI_DIR" +./gencert.sh kojiweb "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=kojiweb/CN=$KOJI_MASTER_FQDN" +./gencert.sh kojihub "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=kojihub/CN=$KOJI_MASTER_FQDN" +./gencert.sh kojiadmin "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=$ORG_UNIT/CN=kojiadmin" +./gencert.sh kojira "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=$ORG_UNIT/CN=kojira" +popd + +# Copy certificates into ~/.koji for kojiadmin +useradd kojiadmin +ADMIN_KOJI_DIR="$(echo ~kojiadmin)"/.koji +mkdir -p "$ADMIN_KOJI_DIR" +cp -f "$KOJI_PKI_DIR"/kojiadmin.pem "$ADMIN_KOJI_DIR"/client.crt +cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_KOJI_DIR"/clientca.crt +cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_KOJI_DIR"/serverca.crt +chown -R kojiadmin:kojiadmin "$ADMIN_KOJI_DIR" + + +## POSTGRESQL SERVER +# Initialize PostgreSQL DB +mkdir -p "$POSTGRES_DIR" +chown -R "$POSTGRES_USER":"$POSTGRES_USER" "$POSTGRES_DIR" +if [[ "$POSTGRES_DIR" != "$POSTGRES_DEFAULT_DIR" ]]; then + if [ "$(ls -A "$POSTGRES_DEFAULT_DIR")" ]; then + mv "$POSTGRES_DEFAULT_DIR" "$POSTGRES_DEFAULT_DIR".old + else + rm -rf "$POSTGRES_DEFAULT_DIR" + fi + ln -sf "$POSTGRES_DIR" "$POSTGRES_DEFAULT_DIR" + chown -h "$POSTGRES_USER":"$POSTGRES_USER" "$POSTGRES_DEFAULT_DIR" +fi +sudo -u "$POSTGRES_USER" initdb --pgdata "$POSTGRES_DEFAULT_DIR"/data +systemctl enable --now postgresql + +# Setup User Accounts +useradd -r koji + +# Setup PostgreSQL and populate schema +sudo -u "$POSTGRES_USER" createuser --no-superuser --no-createrole --no-createdb koji +sudo -u "$POSTGRES_USER" createdb -O koji koji +sudo -u koji psql koji koji < /usr/share/doc/koji*/docs/schema.sql + +# Authorize Koji-web and Koji-hub resources +cat > "$POSTGRES_DEFAULT_DIR"/data/pg_hba.conf <<- EOF +#TYPE DATABASE USER CIDR-ADDRESS METHOD +host koji all 127.0.0.1/32 trust +host koji all ::1/128 trust +local koji all trust +EOF +systemctl reload postgresql + +# Bootstrapping the initial koji admin user into the PostgreSQL database +# SSL Certificate authentication +sudo -u koji psql -c "insert into users (name, status, usertype) values ('kojiadmin', 0, 0);" + +# Give yourself admin permissions +sudo -u koji psql -c "insert into user_perms (user_id, perm_id, creator_id) values (1, 1, 1);" + + +## KOJI CONFIGURATION FILES +# Koji Hub +mkdir -p /etc/koji-hub +cat > /etc/koji-hub/hub.conf <<- EOF +[hub] +DBName = koji +DBUser = koji +KojiDir = $KOJI_DIR +DNUsernameComponent = CN +ProxyDNs = C=$COUNTRY_CODE,ST=$STATE,L=$LOCATION,O=$ORGANIZATION,OU=kojiweb,CN=$KOJI_MASTER_FQDN +LoginCreatesUser = On +KojiWebURL = $KOJI_URL/koji +DisableNotifications = True +EOF + +mkdir -p /etc/httpd/conf.d +cat > /etc/httpd/conf.d/kojihub.conf <<- EOF +Alias /kojihub /usr/share/koji-hub/kojiapp.py + + Options ExecCGI + SetHandler wsgi-script + Require all granted + +Alias /kojifiles "$KOJI_DIR" + + Options Indexes SymLinksIfOwnerMatch + AllowOverride None + Require all granted + + + SSLVerifyClient require + SSLVerifyDepth 10 + SSLOptions +StdEnvVars + +EOF + +# Koji Web +mkdir -p /etc/kojiweb +cat > /etc/kojiweb/web.conf <<- EOF +[web] +SiteName = koji +KojiHubURL = $KOJI_URL/kojihub +KojiFilesURL = $KOJI_URL/kojifiles +WebCert = $KOJI_PKI_DIR/kojiweb.pem +ClientCA = $KOJI_PKI_DIR/koji_ca_cert.crt +KojiHubCA = $KOJI_PKI_DIR/koji_ca_cert.crt +LoginTimeout = 72 +Secret = NITRA_IS_NOT_CLEAR +LibPath = /usr/share/koji-web/lib +LiteralFooter = True +EOF + +mkdir -p /etc/httpd/conf.d +cat > /etc/httpd/conf.d/kojiweb.conf <<- EOF +Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py" + + Options ExecCGI + SetHandler wsgi-script + Require all granted + +Alias /koji-static "/usr/share/koji-web/static" + + Options None + AllowOverride None + Require all granted + +EOF + +# Koji CLI +cat > "$ADMIN_KOJI_DIR"/config <<- EOF +[koji] +server = $KOJI_URL/kojihub +weburl = $KOJI_URL/koji +topurl = $KOJI_URL/kojifiles +topdir = $KOJI_DIR +cert = ~/.koji/client.crt +serverca = ~/.koji/serverca.crt +anon_retry = true +EOF +chown kojiadmin:kojiadmin "$ADMIN_KOJI_DIR"/config + +## KOJI APPLICATION HOSTING +# Koji Filesystem Skeleton +mkdir -p "$KOJI_DIR"/{packages,repos,work,scratch,repos-dist} +chown -R "$HTTPD_USER":"$HTTPD_USER" "$KOJI_DIR" + +## Apache Configuration Files +mkdir -p /etc/httpd/conf.d +cat > /etc/httpd/conf.d/ssl.conf <<- EOF +ServerName $KOJI_MASTER_FQDN + +Listen 443 https + +#SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog + +#SSLSessionCache shmcb:/run/httpd/sslcache(512000) + +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin + + + ErrorLog /var/log/httpd/ssl_error_log + TransferLog /var/log/httpd/ssl_access_log + LogLevel warn + + SSLEngine on + SSLProtocol -all +TLSv1.2 + SSLCipherSuite EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH:EDH+aRSA:HIGH:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS:!RC4:!DH:!SHA1 + SSLHonorCipherOrder on + + SSLCertificateFile $KOJI_PKI_DIR/kojihub.pem + SSLCertificateKeyFile $KOJI_PKI_DIR/private/kojihub.key + SSLCertificateChainFile $KOJI_PKI_DIR/koji_ca_cert.crt + SSLCACertificateFile $KOJI_PKI_DIR/koji_ca_cert.crt + SSLVerifyClient optional + SSLVerifyDepth 10 + + + SSLOptions +StdEnvVars + + + SSLOptions +StdEnvVars + + + CustomLog /var/log/httpd/ssl_request_log "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" + +EOF + +mkdir -p /etc/httpd/conf.modules.d +cat > /etc/httpd/conf.modules.d/wsgi.conf <<- EOF +WSGISocketPrefix /run/httpd/wsgi +EOF +cat > /etc/httpd/conf.modules.d/ssl.conf <<- EOF +LoadModule ssl_module lib/httpd/modules/mod_ssl.so +EOF + +systemctl enable --now httpd + + +## TEST KOJI CONNECTIVITY +sudo -u kojiadmin koji moshimoshi + + +## KOJI DAEMON - BUILDER +# Add the host entry for the koji builder to the database +sudo -u kojiadmin koji add-host "$KOJI_SLAVE_FQDN" "$RPM_ARCH" + +# Add the host to the createrepo channel +sudo -u kojiadmin koji add-host-to-channel "$KOJI_SLAVE_FQDN" createrepo + +# A note on capacity +sudo -u kojiadmin koji edit-host --capacity="$KOJID_CAPACITY" "$KOJI_SLAVE_FQDN" + +# Generate certificates +pushd "$KOJI_PKI_DIR" +./gencert.sh "$KOJI_SLAVE_FQDN" "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/CN=$KOJI_SLAVE_FQDN" +popd + +if [[ "$KOJI_SLAVE_FQDN" = "$KOJI_MASTER_FQDN" ]]; then + "$SCRIPT_DIR"/deploy-koji-builder.sh +fi + + +## KOJIRA - DNF|YUM REPOSITORY CREATION AND MAINTENANCE +# Add the user entry for the kojira user +sudo -u kojiadmin koji add-user kojira +sudo -u kojiadmin koji grant-permission repo kojira + +# Kojira Configuration Files +mkdir -p /etc/kojira +cat > /etc/kojira/kojira.conf <<- EOF +[kojira] +server=$KOJI_URL/kojihub +topdir=$KOJI_DIR +logfile=/var/log/kojira.log +cert = $KOJI_PKI_DIR/kojira.pem +serverca = $KOJI_PKI_DIR/koji_ca_cert.crt +EOF + +# Ensure postgresql is started prior to running kojira service +mkdir -p /etc/systemd/system/kojira.service.d +cat > /etc/systemd/system/kojira.service.d/after-postgresql.conf < /etc/systemd/system/mash.service <<- EOF +[Unit] +Description=Mash script to loop local repository creation for local image builds + +[Service] +User=kojiadmin +Group=kojiadmin +ExecStart=$MASH_SCRIPT_DIR/mash.sh +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target +EOF +systemctl daemon-reload +systemctl enable --now mash diff --git a/koji-setup/deploy-upstreams.sh b/koji-setup/deploy-upstreams.sh new file mode 100755 index 0000000..7e5aff8 --- /dev/null +++ b/koji-setup/deploy-upstreams.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/globals.sh +source "$SCRIPT_DIR"/parameters.sh + +mkdir -p "$UPSTREAMS_DIR" +chown -R "$GIT_USER":"$GIT_USER" "$UPSTREAMS_DIR" +mkdir -p "$HTTPD_DOCUMENT_ROOT" +UPSTREAMS_LINK="$HTTPD_DOCUMENT_ROOT"/"$(basename "$UPSTREAMS_DIR")" +ln -sf "$UPSTREAMS_DIR" "$UPSTREAMS_LINK" +chown -h "$GIT_USER":"$GIT_USER" "$UPSTREAMS_LINK" +usermod -a -G "$GIT_USER" "$HTTPD_USER" diff --git a/koji-setup/gencert.sh b/koji-setup/gencert.sh new file mode 100755 index 0000000..9c6c896 --- /dev/null +++ b/koji-setup/gencert.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +KOJI_USER="$1" +CERT_SUBJECT="$2" + +openssl genrsa -out private/"$KOJI_USER".key 2048 +if [ -z "$CERT_SUBJECT" ]; then + openssl req -config ssl.cnf -new -nodes -out certs/"$KOJI_USER".csr -key private/"$KOJI_USER".key +else + openssl req -subj "$CERT_SUBJECT" -config ssl.cnf -new -nodes -out certs/"$KOJI_USER".csr -key private/"$KOJI_USER".key +fi +openssl ca -batch -config ssl.cnf -keyfile private/koji_ca_cert.key -cert koji_ca_cert.crt -out certs/"$KOJI_USER".crt -outdir certs -infiles certs/"$KOJI_USER".csr +cat certs/"$KOJI_USER".crt private/"$KOJI_USER".key > "$KOJI_USER".pem +# Browser certificate is not password-protected, ask users to change their password +openssl pkcs12 -export -inkey private/"$KOJI_USER".key -in certs/"$KOJI_USER".crt -CAfile koji_ca_cert.crt -out certs/"$KOJI_USER"_browser_cert.p12 -passout pass: diff --git a/koji-setup/globals.sh b/koji-setup/globals.sh new file mode 100644 index 0000000..d7fceb7 --- /dev/null +++ b/koji-setup/globals.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +#### START DO NOT EDIT #### +export GIT_USER=gitolite +export GIT_DEFAULT_DIR=/var/lib/gitolite + +export POSTGRES_USER=postgres +export POSTGRES_DEFAULT_DIR=/var/lib/pgsql + +export HTTPD_USER=httpd +export HTTPD_DOCUMENT_ROOT=/var/www/html + +export KOJI_PKI_DIR=/etc/pki/koji + +check_dependency() { + if [[ "$#" -ne 1 ]]; then + echo "Incorrect number of arguments!" >&2 + exit 1 + fi + if ! type "$1"; then + echo "$1 not found!" >&2 + exit 1 + fi +} + +#### END DO NOT EDIT #### diff --git a/koji-setup/mash.sh b/koji-setup/mash.sh new file mode 100755 index 0000000..e96f8cc --- /dev/null +++ b/koji-setup/mash.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +. /etc/profile.d/proxy.sh || : + +BUILD_ARCH="${BUILD_ARCH:-x86_64}" +KOJI_DIR="${KOJI_DIR:-/srv/koji}" +MASH_DIR="${MASH_DIR:-/srv/mash}" +MASH_TRACKER_FILE="$MASH_DIR"/latest-mash-build +MASH_TRACKER_DIR="$MASH_DIR"/latest +MASH_DIR_OLD="$MASH_TRACKER_DIR".old +MASH_DIR_NEW="$MASH_TRACKER_DIR".new + +create_dist_repos() { + local source_dir="${1}" + local output_dir="${2}" + + local work_dir="$(mktemp -d)" + + local nvr_pkg_list="${work_dir}/nvr-pkg-list" + local bin_rpm_paths="${work_dir}/bin-rpm-paths" + local debuginfo_rpm_paths="${work_dir}/debuginfo-rpm-paths" + local src_rpm_paths="${work_dir}/src-rpm-paths" + local comps_file="${work_dir}/comps.xml" + + sed -r -e 's|[^/]+/||' -e "s|^|${KOJI_DIR}/|" "${KOJI_REPO_PATH}/${BUILD_ARCH}/pkglist" > "${bin_rpm_paths}" + cut -d/ -f3-5 "${KOJI_REPO_PATH}/${BUILD_ARCH}/pkglist" | sort -u > "${nvr_pkg_list}" + while IFS='/' read -r name version release; do + local debuginfo_rpm_path="${KOJI_DIR}/packages/${name}/${version}/${release}/${BUILD_ARCH}/${name}-debuginfo-${version}-${release}.${BUILD_ARCH}.rpm" + if [[ -s "${debuginfo_rpm_path}" ]]; then + echo "${debuginfo_rpm_path}" >> "${debuginfo_rpm_paths}" + fi + echo "${KOJI_DIR}/packages/${name}/${version}/${release}/src/${name}-${version}-${release}.src.rpm" >> "${src_rpm_paths}" + done < "${nvr_pkg_list}" + + cp -f "${KOJI_REPO_PATH}/groups/comps.xml" "${comps_file}" + + make_repo "${source_dir}" "${output_dir}" "clear/${BUILD_ARCH}/os" "Packages" "${bin_rpm_paths}" "${comps_file}" & + make_repo "${source_dir}" "${output_dir}" "clear/${BUILD_ARCH}/debug" "." "${debuginfo_rpm_paths}" & + make_repo "${source_dir}" "${output_dir}" "clear/source/SRPMS" "." "${src_rpm_paths}" & + wait + + create_dnf_conf "${work_dir}/dnf-os.conf" "${output_dir}/clear/${BUILD_ARCH}/os" clear-os + create_dnf_conf "${work_dir}/dnf-debug.conf" "${output_dir}/clear/${BUILD_ARCH}/debug" clear-debug + create_dnf_conf "${work_dir}/dnf-SRPMS.conf" "${output_dir}/clear/source/SRPMS" clear-SRPMS + + write_packages_file "${work_dir}/dnf-os.conf" "$output_dir/clear/$BUILD_ARCH/packages-os" + write_packages_file "${work_dir}/dnf-debug.conf" "$output_dir/clear/$BUILD_ARCH/packages-debug" + write_packages_file "${work_dir}/dnf-SRPMS.conf" "$output_dir/clear/source/packages-SRPMS" + + rm -rf "${work_dir}" +} + +make_repo() { + local previous_repo_dir="${1}/${3}" + local repo_dir="${2}/${3}" + local rpm_dir="${repo_dir}/${4}" + local file_list="${5}" + local comps_file="${6}" + + local create_repo_cmd="createrepo_c --quiet --database --compress-type xz --workers $(nproc --all)" + if [[ -e "${previous_repo_dir}" ]]; then + create_repo_cmd="${create_repo_cmd} --update --update-md-path ${previous_repo_dir}" + fi + + mkdir -p "${rpm_dir}" + xargs -a "${file_list}" -I {} ln -sf {} "${rpm_dir}" + if [[ -z "${comps_file}" ]]; then + ${create_repo_cmd} "${repo_dir}" + else + ${create_repo_cmd} --groupfile "${comps_file}" "${repo_dir}" + fi +} + +create_dnf_conf() { + local dnf_conf="${1}" + local repo_path="${2}" + local repo_name="${3:-clear}" + cat > "${dnf_conf}" < "${output_file}" +} + +if [[ -e "$MASH_TRACKER_FILE" ]]; then + MASH_BUILD_NUM="$(< "$MASH_TRACKER_FILE")" +else + MASH_BUILD_NUM=0 +fi +KOJI_TAG="${KOJI_TAG:-"dist-clear"}" +KOJI_REPO_PATH="$(realpath "$KOJI_DIR/repos/$KOJI_TAG-build/latest")" +KOJI_BUILD_NUM="$(basename "$KOJI_REPO_PATH")" +if [[ "$MASH_BUILD_NUM" -ne "$KOJI_BUILD_NUM" ]]; then + rm -rf "$MASH_DIR_NEW" + mkdir -p "$MASH_DIR_NEW" + create_dist_repos "$MASH_TRACKER_DIR" "$MASH_DIR_NEW" + if [[ -e "$MASH_TRACKER_DIR" ]]; then + mv "$MASH_TRACKER_DIR" "$MASH_DIR_OLD" + fi + mv "$MASH_DIR_NEW" "$MASH_TRACKER_DIR" + rm -rf "$MASH_DIR_OLD" + + echo "$KOJI_BUILD_NUM" > "$MASH_TRACKER_FILE" +fi diff --git a/koji-setup/parameters.sh b/koji-setup/parameters.sh new file mode 100644 index 0000000..e5ef4f6 --- /dev/null +++ b/koji-setup/parameters.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +## KOJI RPM BUILD AND TRACKER +export KOJI_DIR=/srv/koji +export KOJI_MOUNT_DIR=/mnt/koji +export KOJI_MASTER_FQDN="$(hostname -f)" +export KOJI_SLAVE_FQDN="$KOJI_MASTER_FQDN" +export KOJI_URL=https://"$KOJI_MASTER_FQDN" +export KOJID_CAPACITY=16 +export TAG_NAME=clear +# Use for koji SSL certificates +export COUNTRY_CODE='EX' +export STATE='Example' +export LOCATION='Example' +export ORGANIZATION='Example' +export ORG_UNIT='Example' +# Use for importing existing RPMs +export RPM_ARCH='x86_64' +export SRC_RPM_DIR= +export BIN_RPM_DIR= +export DEBUG_RPM_DIR= +# Comment the following if supplying all RPMs as an upstream and not a downstream +export EXTERNAL_REPO=https://cdn.download.clearlinux.org/releases/"$(curl https://download.clearlinux.org/latest)"/clear/\$arch/os/ + +## POSTGRESQL DATABASE +export POSTGRES_DIR=/srv/pgsql + +## GIT REPOSITORIES +export GIT_DIR=/srv/gitolite +export GIT_FQDN="$KOJI_MASTER_FQDN" +export IS_ANONYMOUS_GIT_NEEDED=false +export GITOLITE_PUB_KEY='' + +## UPSTREAMS CACHE +export UPSTREAMS_DIR=/srv/upstreams + +## MASH RPMS +export MASH_DIR=/srv/mash +export MASH_SCRIPT_DIR=/usr/local/bin diff --git a/koji.yaml b/koji.yaml new file mode 100644 index 0000000..e843fd7 --- /dev/null +++ b/koji.yaml @@ -0,0 +1,38 @@ +--- +- hosts: koji-all-in-one + tasks: + - name: Add required bundles for koji + swupd: + name: koji postgresql web-server-basic + state: present + become: true + - name: copy koji-setup scripts + copy: + src: koji-setup + dest: /tmp/ + owner: "{{ ansible_ssh_user }}" + group: "{{ ansible_ssh_user }}" + - name: update bootstrap-build file permissions + file: + path: /tmp/koji-setup/bootstrap-build.sh + mode: 0755 + - name: update deploy-koji file permissions + file: + path: /tmp/koji-setup/deploy-koji.sh + mode: 0755 + - name: update gencert file permissions + file: + path: /tmp/koji-setup/gencert.sh + mode: 0755 + - name: run koji deployment + command: + ./deploy-koji.sh + become: true + args: + chdir: /tmp/koji-setup + - name: bootstrap build tags and targets + command: + ./bootstrap-build.sh + become: true + args: + chdir: /tmp/koji-setup