smeserver-pungi/build_scripts/jenkins.py

#!/usr/bin/env python3
# coding=utf-8

"""
This script should be inserted to jenkins job
"""
import argparse
import logging
import os
import signal
import subprocess
from distutils.util import strtobool
from pathlib import Path
from typing import (
    List,
    Union,
    Optional,
)

import requests

logging.basicConfig(level=logging.INFO)


def signal_handler(signum, frame, process: subprocess.Popen):
    logging.info(
        'Processing signal "%s" in frame "%s"',
        signal.Signals(signum),
        frame,
    )
    if process.poll() is None:
        process.send_signal(signum)


class Runner:

    def __init__(
            self,
            working_root_directory: Path,
            product_name: str,
            distribution_major_version: int,
            distribution_minor_version: int,
            arch: str,
            branch: str,
            keep_builds: int,
            use_products_repos: bool,
            env_files: List[str],
            beta_suffix: str,
            sigkeys_fingerprints: List[str],
            skip_mirroring: bool,
            local_repos: List[str],
            not_needed_variant: str,
            pgp_sign_keyid: str,
            git_url: str,
            git_project: str,
            git_type: str,
            git_auth_token: str,
            git_auth_username: str,
            sign_service_username: str,
            sign_service_password: str,
            sign_service_endpoint: str,
            koji_excluded_packages: List[str],
    ):
        self.sign_service_username = sign_service_username
        self.sign_service_password = sign_service_password
        self.sign_service_endpoint = sign_service_endpoint
        self.git_url = git_url
        self.git_project = git_project
        self.git_type = git_type
        self.git_auth_username = git_auth_username
        self.git_auth_token = git_auth_token
        self.working_root_directory = working_root_directory
        self.product_name = product_name
        self.distribution_major_version = distribution_major_version
        self.distribution_minor_version = distribution_minor_version
        self.arch = arch
        self.branch = branch
        self.keep_builds = keep_builds
        self.compose_dir = 'last_compose_dir'
        self.use_products_repos = use_products_repos
        self.env_files = env_files
        self.beta_suffix = beta_suffix
        self.sigkeys_fingerprints = sigkeys_fingerprints
        self.skip_mirroring = skip_mirroring
        self.local_repos = local_repos
        self.not_needed_variant = not_needed_variant
        self.pgp_sign_keyid = pgp_sign_keyid
        self.repos_folder = working_root_directory.joinpath(
            f'alma-{distribution_major_version}-{arch}'
        )
        self.koji_excluded_packages = koji_excluded_packages

        self.build_scripts_path = working_root_directory.joinpath(
            'pungi-scripts-public',
            'build_scripts',
        )
        self.env_path = working_root_directory.joinpath(
            f'{self.product_name}{self.distribution_major_version}{self.arch}'
        )
        self.koji_profile_name = (
            f'{self.product_name.lower()}_{self.distribution_major_version}'
        )
        if self.beta_suffix:
            self.pungi_label = (
                f'Beta-{self.distribution_major_version}.'
                f'{self.distribution_minor_version}'
            )
        else:
            self.pungi_label = (
                f'Update-{self.distribution_major_version}.'
                f'{self.distribution_minor_version}'
            )
        self.final_repo_folders = ' '.join(self.get_variants(
            arch=self.arch,
            distribution_major_version=self.distribution_major_version,
            distribution_minor_version=self.distribution_minor_version,
            beta_suffix=self.beta_suffix,
        ))
        self.pungi_configs_git_repo = (
            'https://github.com/AlmaLinux/pungi-scripts-public.git'
        )
        self.compose_dir_full_path = self.env_path.joinpath(
            'pungi-results',
            self.compose_dir,
        )

    @staticmethod
    def run_command(
            command: str,
            exit_or_not: bool = True,
            raise_exception: bool = True,
            use_sudo: bool = True
    ) -> None:

        sudo_suffix = 'sudo' if use_sudo else ''
        cmd_line = f"{sudo_suffix} bash -c \"{command}\""
        logging.info(cmd_line)
        process = subprocess.Popen(
            cmd_line,
            shell=True,
            executable='/bin/bash',
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            universal_newlines=True,
        )
        for _signum in (signal.SIGTERM, signal.SIGINT):
            signal.signal(
                _signum,
                lambda signum, frame: signal_handler(
                    signum=signum,
                    frame=frame,
                    process=process,
                ),
            )

        while process.poll() is None:
            if process.stdout is not None:
                realtime_stdout_line = process.stdout.readline().strip()
                if realtime_stdout_line:
                    print(realtime_stdout_line, flush=True)
        else:
            if process.poll():
                if exit_or_not:
                    exit(process.poll())
                elif raise_exception:
                    raise subprocess.SubprocessError()

    def cleanup(self):
        command = f'cd {self.build_scripts_path} && /usr/bin/env ' \
                  f'python3 cleanup.py ' \
                  f'--env-path {self.env_path} ' \
                  f'--keep-builds {self.keep_builds} '
        if self.compose_dir is not None:
            command = f'{command} --excluded-dirs {self.compose_dir}'
        self.run_command(command=command)

    def enable_products_repos(self):
        suffix = 'products_repos'
        target = Path(f'{self.repos_folder}').joinpath(suffix)
        source = Path(f'{self.repos_folder}-{suffix}')
        if self.use_products_repos:
            logging.info('Enable products repos')
            if not target.is_symlink():
                target.symlink_to(source)
        elif target.exists() and target.is_symlink():
            logging.info('Disable products repos')
            target.unlink(missing_ok=True)

    def prepare(self):

        if not self.skip_mirroring:
            command = (
                f'cd {self.build_scripts_path} && /usr/bin/env '
                f'python3 prepare.py '
                f'--env-path {self.env_path} '
                f'dnf_reposync_synchronize '
                f'{"--use-products-repos " if self.use_products_repos else ""}'
                f'--mirroring-target {self.repos_folder} '
                f'--product-name {self.product_name} '
                f'--arch {self.arch} '
                '--distribution-major-version '
                f'{self.distribution_major_version}'
            )
            self.run_command(command=command)
        self.enable_products_repos()
        command = (
            f'cd {self.build_scripts_path} && /usr/bin/env '
            f'python3 prepare.py '
            f'--env-path {self.env_path} '
            f'add_env_files '
            f'--env-files {" ".join(self.env_files)}'
        )
        self.run_command(command=command)

        command = (
            f'cd {self.build_scripts_path} && /usr/bin/env '
            f'python3 prepare.py '
            f'--env-path {self.env_path} '
            f'add_koji_profile '
            f'--koji-profile-name {self.koji_profile_name}'
        )
        self.run_command(command=command)

        command = (
            f'cd {self.build_scripts_path} && /usr/bin/env '
            f'python3 prepare.py '
            f'--env-path {self.env_path} '
            f'prepare_build_conf '
            f'--product-name {self.product_name} '
            f'--arch {self.arch} '
            f'--distribution-major-version {self.distribution_major_version} '
            f'--distribution-minor-version {self.distribution_minor_version} '
            f'--beta-suffix={self.beta_suffix} '
            f'--sigkeys-fingerprints {" ".join(self.sigkeys_fingerprints)} '
            f'--git-url {self.git_url} '
            f'--git-project {self.git_project} '
            f'--git-type {self.git_type} '
            f'--git-auth-token {self.git_auth_token} '
            f'--git-auth-username {self.git_auth_username}'
        )
        self.run_command(command=command)

    def build(self):
        # remove an old compose dir

        command = (
            f'[[ -d "{self.compose_dir_full_path}" ]] && '
            f'rm -rf {self.compose_dir_full_path}'
        )
        self.run_command(command, exit_or_not=False, raise_exception=False)

        command = (
            f'cd {self.build_scripts_path} && /usr/bin/env '
            f'python3 build.py '
            f'--env-path {self.env_path} '
            f'--local-mirror-path {self.repos_folder} '
            f'--pungi-label {self.pungi_label} '
            f'--result-directory {self.compose_dir} '
            f'--local-repos {" ".join(self.local_repos)} '
            f'--koji-excluded-packages {" ".join(self.koji_excluded_packages)}'
        )
        self.run_command(command=command)

    def post(self):
        command = (
            f'cd {self.build_scripts_path} && /usr/bin/env '
            f'python3 post_actions.py '
            f'--env-path {self.env_path} '
            f'--arch {self.arch} '
            f'--source-repos-folder {self.repos_folder} '
            f'--repos {self.final_repo_folders} '
            f'--not-needed-repos {self.not_needed_variant} '
        )
        if self.pgp_sign_keyid:
            command += f'--pgp-sign-keyid={self.pgp_sign_keyid} '
        if self.sign_service_username:
            f'--sign-service-username={self.sign_service_username} '
        if self.sign_service_password:
            f'--sign-service-password={self.sign_service_password} '
        if self.sign_service_endpoint:
            f'--sign-service-endpoint={self.sign_service_endpoint} '
        self.run_command(command=command)

    @staticmethod
    def get_variants(
            arch: str,
            distribution_major_version: int,
            distribution_minor_version: int,
            beta_suffix: str,
    ) -> List[str]:
        url = (
            'https://git.almalinux.org/almalinux/pungi-almalinux/raw/branch/'
            f'a{distribution_major_version}.{distribution_minor_version}'
            f'{beta_suffix}/{arch}/variants_options.json'
        )
        response = requests.get(url)
        response.raise_for_status()
        variants_data = response.json()  # type: dict
        return list(variants_data.keys())

    def checkout_scripts(self):

        pungi_configs_folder = self.working_root_directory.joinpath(
            'pungi-scripts-public'
        )
        try:
            command = f'ls {pungi_configs_folder}'
            self.run_command(command=command, exit_or_not=False)
            command = (
                f'cd {pungi_configs_folder} && '
                'rm -rf ./* && git checkout -- . && '
                'git fetch && git clean -f && '
                f'git checkout -B {self.branch} origin/{self.branch}'
            )
            self.run_command(command=command)
        except subprocess.SubprocessError:
            command = (
                f'cd {self.working_root_directory} && '
                f'git clone "{self.pungi_configs_git_repo}" '
                f'&& cd {pungi_configs_folder} '
                f'&& git checkout -B {self.branch} origin/{self.branch}'
            )
            self.run_command(command=command)


class StoreAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        if values is None or not values:
            raise argparse.ArgumentError(
                self,
                f'Invalid value: {values}',
            )
        setattr(namespace, self.dest, values)


def get_env_var(
        key: str,
        default: Optional[Union[str, int, List]] = None,
        is_bool: bool = False,
        is_multiline: bool = False,
) -> Union[str, bool, int, List[str]]:
    result = os.environ.get(key.lower()) or os.environ.get(key.upper())
    result = result or default
    if is_bool:
        result = strtobool(result)
    if is_multiline:
        result = list(filter(None, result.split('\n')))
    return result


def create_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--working-root-directory',
        action=StoreAction,
        default=get_env_var(key='remote_home_dir'),
        type=Path,
    )
    parser.add_argument(
        '--pgp-sign-keyid',
        action=StoreAction,
        default=get_env_var(
            key='pgp_sign_keyid',
            default='',
        ),
        type=str,
    )
    parser.add_argument(
        '--arch',
        action=StoreAction,
        default=get_env_var(key='arch'),
        type=str,
    )
    parser.add_argument(
        '--product-name',
        action=StoreAction,
        default=get_env_var(key='product_name'),
        type=str,
    )
    parser.add_argument(
        '--distribution-major-version',
        action=StoreAction,
        default=get_env_var(key='distribution_major_version'),
        type=int,
    )
    parser.add_argument(
        '--distribution-minor-version',
        action=StoreAction,
        default=get_env_var(key='distribution_minor_version'),
        type=int,
    )
    parser.add_argument(
        '--beta-suffix',
        action='store',
        default=get_env_var(key='beta_suffix', default=''),
        type=str,
    )
    parser.add_argument(
        '--not-needed-variant',
        action=StoreAction,
        default=get_env_var(key='not_needed_variant', default='Minimal'),
        type=str,
    )
    parser.add_argument(
        '--keep-builds',
        action=StoreAction,
        type=int,
        default=get_env_var(key='keep_builds', default=1),
    )
    parser.add_argument(
        '--env-files',
        action='store',
        type=List[str],
        default=get_env_var(
            key='add_env_files',
            default='',
            is_multiline=True,
        ),
        nargs='+',
    )
    parser.add_argument(
        '--local-repos',
        action='store',
        type=List[str],
        default=get_env_var(
            key='local_repos',
            default='',
            is_multiline=True,
        ),
        nargs='+',
    )
    parser.add_argument(
        '--sigkeys-fingerprints',
        action=StoreAction,
        default=get_env_var(
            key='sigkeys_fingerprints',
            default='',
            is_multiline=True,
        ),
        nargs='+',
    )
    parser.add_argument(
        '--skip-mirroring',
        action='store_true',
        default=get_env_var(
            key='skip_mirroring',
            default='False',
            is_bool=True,
        ),
    )
    parser.add_argument(
        '--use-products-repos',
        action='store_true',
        default=get_env_var(
            key='use_products_repos',
            default='False',
            is_bool=True,
        ),
    )
    parser.add_argument(
        '--git-auth-token',
        action='store',
        type=str,
        default=get_env_var(key='git_auth_token'),
    )
    parser.add_argument(
        '--git-auth-username',
        action='store',
        type=str,
        default=get_env_var(key='git_auth_username'),
    )
    parser.add_argument(
        '--git-url',
        action='store',
        type=str,
        default=get_env_var(
            key='git_storage_url',
            default='git.almalinux.org',
        ),
    )
    parser.add_argument(
        '--git-project',
        action='store',
        type=str,
        default=get_env_var(
            key='git_project',
            default='almalinux/pungi-almalinux',
        ),
    )
    parser.add_argument(
        '--git-type',
        action='store',
        type=str,
        default=get_env_var(key='git_type', default='gitea'),
    )
    parser.add_argument(
        '--sign-service-username',
        action='store',
        help='An username of a sign service',
        default=get_env_var(
            key='sign_service_username',
            default='',
        ),
    )
    parser.add_argument(
        '--sign-service-password',
        action='store',
        help='A password of a sign service',
        default=get_env_var(
            key='sign_service_password',
            default='',
        ),
    )
    parser.add_argument(
        '--sign-service-endpoint',
        action='store',
        help='An endpoint of a sign service',
        default=get_env_var(
            key='sign_service_endpoint',
            default='',
        ),
    )
    parser.add_argument(
        '-e',
        '--koji-excluded-packages',
        required=False,
        nargs='+',
        type=List[str],
        default=get_env_var(
            key='koji_excluded_packages',
            default='',
            is_multiline=True,
        ),
    )
    parser.add_argument(
        '--branch',
        action=StoreAction,
        type=str,
        default=get_env_var(key='branch', default='master'),
    )
    return parser


def main():

    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)

    sensitive_arguments = (
        'git_auth_token',
        'sign_service_password',
    )
    args = create_parser().parse_args()
    logging.info('All CLI arguments:')
    for arg_name, arg_value in vars(args).items():
        if arg_name in sensitive_arguments:
            continue
        logging.info('%s: %s', arg_name, arg_value)
    runner = Runner(
        working_root_directory=args.working_root_directory,
        product_name=args.product_name,
        distribution_major_version=args.distribution_major_version,
        distribution_minor_version=args.distribution_minor_version,
        arch=args.arch,
        branch=args.branch,
        keep_builds=args.keep_builds,
        use_products_repos=args.use_products_repos,
        env_files=args.env_files,
        beta_suffix=args.beta_suffix,
        sigkeys_fingerprints=args.sigkeys_fingerprints,
        skip_mirroring=args.skip_mirroring,
        local_repos=args.local_repos,
        not_needed_variant=args.not_needed_variant,
        pgp_sign_keyid=args.pgp_sign_keyid,
        git_url=args.git_url,
        git_project=args.git_project,
        git_auth_token=args.git_auth_token,
        git_auth_username=args.git_auth_username,
        git_type=args.git_type,
        sign_service_username=args.sign_service_username,
        sign_service_password=args.sign_service_password,
        sign_service_endpoint=args.sign_service_endpoint,
        koji_excluded_packages=args.koji_excluded_packages,
    )
    try:
        runner.checkout_scripts()
        runner.prepare()
        runner.build()
        runner.post()
    finally:
        runner.cleanup()


if __name__ == '__main__':
    main()