add almalinux build scripts

build_scripts/build.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+import argparse
+import logging
+import os
+import subprocess
+from pathlib import Path
+from shutil import rmtree
+from typing import (
+    Optional,
+    List,
+)
+
+PUNGI_RESULTS = 'pungi-results'
+
+logging.basicConfig(level=logging.INFO)
+
+
+def prepare_koji_env(
+        env_path: str,
+        local_mirror: str,
+        local_repos: List[str],
+        koji_excluded_packages: List[str],
+):
+    koji_env_path = os.path.join(
+        env_path,
+        'koji',
+    )
+    logging.info(
+        'Update koji env in "%s"',
+        koji_env_path,
+    )
+    if os.path.exists(koji_env_path):
+        rmtree(koji_env_path)
+    os.makedirs(koji_env_path, exist_ok=True)
+
+    command = (
+        f'pungi-gather-rpms -p {local_mirror} -t {koji_env_path} '
+        f'-e={" ".join(koji_excluded_packages)}'
+    )
+    logging.info(command)
+    subprocess.check_call(
+        command,
+        shell=True,
+    )
+    if local_repos:
+        local_repos_paths = ' '.join(
+            path for local_repo in local_repos for
+            path in map(str, Path(local_mirror).glob(local_repo))
+        )
+        part_of_command = f'-rd {local_repos_paths}'
+    else:
+        part_of_command = f'-rp {local_mirror}'
+    command = f'pungi-gather-modules {part_of_command} -t {koji_env_path}'
+    logging.info(command)
+    subprocess.check_call(
+        command,
+        shell=True,
+    )
+
+
+def run_build(
+        env_path: str,
+        pungi_label: str,
+        result_directory: Optional[str] = None,
+):
+    logging.info('Run building of distribution')
+    pungi_config_name = 'pungi-build.conf'
+    command = f'pungi-koji --config {pungi_config_name} --label {pungi_label}'
+    if 'Beta-' in pungi_label:
+        command += ' --test'
+    else:
+        command += ' --production'
+    if result_directory is not None:
+        pungi_results_dir_full_path = os.path.join(
+            env_path,
+            PUNGI_RESULTS,
+        )
+        os.makedirs(pungi_results_dir_full_path, exist_ok=True)
+        result_dir_full_path = os.path.join(
+            pungi_results_dir_full_path,
+            result_directory,
+        )
+        command += f' --compose-dir {result_dir_full_path}'
+    else:
+        command += f' --target-dir {PUNGI_RESULTS} --no-latest-link'
+
+    logging.info(command)
+    subprocess.check_call(
+        command,
+        shell=True,
+        cwd=env_path,
+    )
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--env-path',
+        action='store',
+        help='A path to folder which will be used '
+             'for building new distribution',
+        required=True,
+    )
+    parser.add_argument(
+        '--local-mirror-path',
+        action='store',
+        help='A path to local mirror of repos',
+        required=True,
+    )
+    parser.add_argument(
+        '--local-repos',
+        action='store',
+        nargs='*',
+        default=[],
+        type=str,
+        help='List of the local repos in `--local-mirror-path`'
+    )
+    parser.add_argument(
+        '--pungi-label',
+        action='store',
+        help='A label of an build distribution',
+        required=True,
+    )
+    parser.add_argument(
+        '--result-directory',
+        action='store',
+        help='A path to store the result of building',
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '--koji-excluded-packages',
+        required=False,
+        nargs='*',
+        type=str,
+        default=[],
+    )
+    return parser
+
+
+def cli_main():
+
+    args = create_parser().parse_args()
+    os.makedirs(
+        os.path.join(
+            args.env_path,
+            PUNGI_RESULTS,
+        ),
+        exist_ok=True,
+    )
+    prepare_koji_env(
+        env_path=args.env_path,
+        local_mirror=args.local_mirror_path,
+        local_repos=args.local_repos,
+        koji_excluded_packages=args.koji_excluded_packages,
+    )
+    if args.result_directory is not None and \
+            os.path.exists(args.result_directory):
+        rmtree(args.result_directory)
+    run_build(
+        env_path=args.env_path,
+        pungi_label=args.pungi_label,
+        result_directory=args.result_directory,
+    )
+
+
+if __name__ == '__main__':
+    cli_main()

build_scripts/cleanup.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+import argparse
+import logging
+import os
+from pathlib import Path
+from shutil import rmtree
+
+PUNGI_RESULTS = 'pungi-results'
+
+logging.basicConfig(level=logging.INFO)
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--env-path',
+        action='store',
+        help='A path to folder which will be used '
+             'for building new distribution',
+        required=True,
+    )
+    parser.add_argument(
+        '--keep-builds',
+        action='store',
+        help='An amount of kept old builds',
+        required=True,
+        type=int,
+    )
+    parser.add_argument(
+        '--excluded-dirs',
+        help='The list of excluded for deleting dirs',
+        required=False,
+        nargs='+',
+        type=str,
+        default=[],
+    )
+    return parser
+
+
+def cli_main():
+
+    args = create_parser().parse_args()
+    pungi_results_path = os.path.join(
+        args.env_path,
+        PUNGI_RESULTS,
+    )
+    dirs_prefixes = [
+        'latest-',
+        'minimal_iso',
+    ]
+    dirs_prefixes.extend(
+        args.excluded_dirs,
+    )
+    old_pungi_results = sorted(
+        filter(
+            lambda i: not any(
+                i.name.startswith(dir_prefix) for dir_prefix in dirs_prefixes
+            ),
+            filter(
+                lambda i: i.is_dir(),
+                Path(pungi_results_path).iterdir()
+            )
+        ),
+        key=os.path.getmtime,
+    )
+    if args.keep_builds:
+        old_pungi_results = old_pungi_results[:-args.keep_builds]
+    for old_pungi_result in old_pungi_results:
+        logging.info(
+            'Remove old build by path "%s"',
+            old_pungi_result,
+        )
+        rmtree(old_pungi_result)
+
+
+if __name__ == '__main__':
+    cli_main()

build_scripts/jenkins.py

@@ -0,0 +1,595 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+This script should be inserted to jenkins job
+"""
+import argparse
+import logging
+import os
+import signal
+import subprocess
+from distutils.util import strtobool
+from pathlib import Path
+from typing import (
+    List,
+    Union,
+    Optional,
+)
+
+import requests
+
+logging.basicConfig(level=logging.INFO)
+
+
+def signal_handler(signum, frame, process: subprocess.Popen):
+    logging.info(
+        'Processing signal "%s" in frame "%s"',
+        signal.Signals(signum),
+        frame,
+    )
+    if process.poll() is None:
+        process.send_signal(signum)
+
+
+class Runner:
+
+    def __init__(
+            self,
+            working_root_directory: Path,
+            product_name: str,
+            distribution_major_version: int,
+            distribution_minor_version: int,
+            arch: str,
+            branch: str,
+            keep_builds: int,
+            use_products_repos: bool,
+            env_files: List[str],
+            beta_suffix: str,
+            sigkeys_fingerprints: List[str],
+            skip_mirroring: bool,
+            local_repos: List[str],
+            not_needed_variant: str,
+            pgp_sign_keyid: str,
+            git_url: str,
+            git_project: str,
+            git_type: str,
+            git_auth_token: str,
+            git_auth_username: str,
+            sign_service_username: str,
+            sign_service_password: str,
+            sign_service_endpoint: str,
+            koji_excluded_packages: List[str],
+    ):
+        self.sign_service_username = sign_service_username
+        self.sign_service_password = sign_service_password
+        self.sign_service_endpoint = sign_service_endpoint
+        self.git_url = git_url
+        self.git_project = git_project
+        self.git_type = git_type
+        self.git_auth_username = git_auth_username
+        self.git_auth_token = git_auth_token
+        self.working_root_directory = working_root_directory
+        self.product_name = product_name
+        self.distribution_major_version = distribution_major_version
+        self.distribution_minor_version = distribution_minor_version
+        self.arch = arch
+        self.branch = branch
+        self.keep_builds = keep_builds
+        self.compose_dir = 'last_compose_dir'
+        self.use_products_repos = use_products_repos
+        self.env_files = env_files
+        self.beta_suffix = beta_suffix
+        self.sigkeys_fingerprints = sigkeys_fingerprints
+        self.skip_mirroring = skip_mirroring
+        self.local_repos = local_repos
+        self.not_needed_variant = not_needed_variant
+        self.pgp_sign_keyid = pgp_sign_keyid
+        self.repos_folder = working_root_directory.joinpath(
+            f'alma-{distribution_major_version}-{arch}'
+        )
+        self.koji_excluded_packages = koji_excluded_packages
+
+        self.build_scripts_path = working_root_directory.joinpath(
+            'pungi-scripts-public',
+            'build_scripts',
+        )
+        self.env_path = working_root_directory.joinpath(
+            f'{self.product_name}{self.distribution_major_version}{self.arch}'
+        )
+        self.koji_profile_name = (
+            f'{self.product_name.lower()}_{self.distribution_major_version}'
+        )
+        if self.beta_suffix:
+            self.pungi_label = (
+                f'Beta-{self.distribution_major_version}.'
+                f'{self.distribution_minor_version}'
+            )
+        else:
+            self.pungi_label = (
+                f'Update-{self.distribution_major_version}.'
+                f'{self.distribution_minor_version}'
+            )
+        self.final_repo_folders = ' '.join(self.get_variants(
+            arch=self.arch,
+            distribution_major_version=self.distribution_major_version,
+            distribution_minor_version=self.distribution_minor_version,
+            beta_suffix=self.beta_suffix,
+        ))
+        self.pungi_configs_git_repo = (
+            'https://github.com/AlmaLinux/pungi-scripts-public.git'
+        )
+        self.compose_dir_full_path = self.env_path.joinpath(
+            'pungi-results',
+            self.compose_dir,
+        )
+
+    @staticmethod
+    def run_command(
+            command: str,
+            exit_or_not: bool = True,
+            raise_exception: bool = True,
+            use_sudo: bool = True
+    ) -> None:
+
+        sudo_suffix = 'sudo' if use_sudo else ''
+        cmd_line = f"{sudo_suffix} bash -c \"{command}\""
+        logging.info(cmd_line)
+        process = subprocess.Popen(
+            cmd_line,
+            shell=True,
+            executable='/bin/bash',
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
+        for _signum in (signal.SIGTERM, signal.SIGINT):
+            signal.signal(
+                _signum,
+                lambda signum, frame: signal_handler(
+                    signum=signum,
+                    frame=frame,
+                    process=process,
+                ),
+            )
+
+        while process.poll() is None:
+            if process.stdout is not None:
+                realtime_stdout_line = process.stdout.readline().strip()
+                if realtime_stdout_line:
+                    print(realtime_stdout_line, flush=True)
+        else:
+            if process.poll():
+                if exit_or_not:
+                    exit(process.poll())
+                elif raise_exception:
+                    raise subprocess.SubprocessError()
+
+    def cleanup(self):
+        command = f'cd {self.build_scripts_path} && /usr/bin/env ' \
+                  f'python3 cleanup.py ' \
+                  f'--env-path {self.env_path} ' \
+                  f'--keep-builds {self.keep_builds} '
+        if self.compose_dir is not None:
+            command = f'{command} --excluded-dirs {self.compose_dir}'
+        self.run_command(command=command)
+
+    def enable_products_repos(self):
+        suffix = 'products_repos'
+        target = Path(f'{self.repos_folder}').joinpath(suffix)
+        source = Path(f'{self.repos_folder}-{suffix}')
+        if self.use_products_repos:
+            logging.info('Enable products repos')
+            if not target.is_symlink():
+                target.symlink_to(source)
+        elif target.exists() and target.is_symlink():
+            logging.info('Disable products repos')
+            target.unlink(missing_ok=True)
+
+    def prepare(self):
+
+        if not self.skip_mirroring:
+            command = (
+                f'cd {self.build_scripts_path} && /usr/bin/env '
+                f'python3 prepare.py '
+                f'--env-path {self.env_path} '
+                f'dnf_reposync_synchronize '
+                f'{"--use-products-repos " if self.use_products_repos else ""}'
+                f'--mirroring-target {self.repos_folder} '
+                f'--product-name {self.product_name} '
+                f'--arch {self.arch} '
+                '--distribution-major-version '
+                f'{self.distribution_major_version}'
+            )
+            self.run_command(command=command)
+        self.enable_products_repos()
+        command = (
+            f'cd {self.build_scripts_path} && /usr/bin/env '
+            f'python3 prepare.py '
+            f'--env-path {self.env_path} '
+            f'add_env_files '
+            f'--env-files {" ".join(self.env_files)}'
+        )
+        self.run_command(command=command)
+
+        command = (
+            f'cd {self.build_scripts_path} && /usr/bin/env '
+            f'python3 prepare.py '
+            f'--env-path {self.env_path} '
+            f'add_koji_profile '
+            f'--koji-profile-name {self.koji_profile_name}'
+        )
+        self.run_command(command=command)
+
+        command = (
+            f'cd {self.build_scripts_path} && /usr/bin/env '
+            f'python3 prepare.py '
+            f'--env-path {self.env_path} '
+            f'prepare_build_conf '
+            f'--product-name {self.product_name} '
+            f'--arch {self.arch} '
+            f'--distribution-major-version {self.distribution_major_version} '
+            f'--distribution-minor-version {self.distribution_minor_version} '
+            f'--beta-suffix={self.beta_suffix} '
+            f'--sigkeys-fingerprints {" ".join(self.sigkeys_fingerprints)} '
+            f'--git-url {self.git_url} '
+            f'--git-project {self.git_project} '
+            f'--git-type {self.git_type} '
+            f'--git-auth-token {self.git_auth_token} '
+            f'--git-auth-username {self.git_auth_username}'
+        )
+        self.run_command(command=command)
+
+    def build(self):
+        # remove an old compose dir
+
+        command = (
+            f'[[ -d "{self.compose_dir_full_path}" ]] && '
+            f'rm -rf {self.compose_dir_full_path}'
+        )
+        self.run_command(command, exit_or_not=False, raise_exception=False)
+
+        command = (
+            f'cd {self.build_scripts_path} && /usr/bin/env '
+            f'python3 build.py '
+            f'--env-path {self.env_path} '
+            f'--local-mirror-path {self.repos_folder} '
+            f'--pungi-label {self.pungi_label} '
+            f'--result-directory {self.compose_dir} '
+            f'--local-repos {" ".join(self.local_repos)} '
+            f'--koji-excluded-packages {" ".join(self.koji_excluded_packages)}'
+        )
+        self.run_command(command=command)
+
+    def post(self):
+        command = (
+            f'cd {self.build_scripts_path} && /usr/bin/env '
+            f'python3 post_actions.py '
+            f'--env-path {self.env_path} '
+            f'--arch {self.arch} '
+            f'--source-repos-folder {self.repos_folder} '
+            f'--repos {self.final_repo_folders} '
+            f'--not-needed-repos {self.not_needed_variant} '
+        )
+        if self.pgp_sign_keyid:
+            command += f'--pgp-sign-keyid={self.pgp_sign_keyid} '
+        if self.sign_service_username:
+            f'--sign-service-username={self.sign_service_username} '
+        if self.sign_service_password:
+            f'--sign-service-password={self.sign_service_password} '
+        if self.sign_service_endpoint:
+            f'--sign-service-endpoint={self.sign_service_endpoint} '
+        self.run_command(command=command)
+
+    @staticmethod
+    def get_variants(
+            arch: str,
+            distribution_major_version: int,
+            distribution_minor_version: int,
+            beta_suffix: str,
+    ) -> List[str]:
+        url = (
+            'https://git.almalinux.org/almalinux/pungi-almalinux/raw/branch/'
+            f'a{distribution_major_version}.{distribution_minor_version}'
+            f'{beta_suffix}/{arch}/variants_options.json'
+        )
+        response = requests.get(url)
+        response.raise_for_status()
+        variants_data = response.json()  # type: dict
+        return list(variants_data.keys())
+
+    def checkout_scripts(self):
+
+        pungi_configs_folder = self.working_root_directory.joinpath(
+            'pungi-scripts-public'
+        )
+        try:
+            command = f'ls {pungi_configs_folder}'
+            self.run_command(command=command, exit_or_not=False)
+            command = (
+                f'cd {pungi_configs_folder} && '
+                'rm -rf ./* && git checkout -- . && '
+                'git fetch && git clean -f && '
+                f'git checkout -B {self.branch} origin/{self.branch}'
+            )
+            self.run_command(command=command)
+        except subprocess.SubprocessError:
+            command = (
+                f'cd {self.working_root_directory} && '
+                f'git clone "{self.pungi_configs_git_repo}" '
+                f'&& cd {pungi_configs_folder} '
+                f'&& git checkout -B {self.branch} origin/{self.branch}'
+            )
+            self.run_command(command=command)
+
+
+class StoreAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if values is None or not values:
+            raise argparse.ArgumentError(
+                self,
+                f'Invalid value: {values}',
+            )
+        setattr(namespace, self.dest, values)
+
+
+def get_env_var(
+        key: str,
+        default: Optional[Union[str, int, List]] = None,
+        is_bool: bool = False,
+        is_multiline: bool = False,
+) -> Union[str, bool, int, List[str]]:
+    result = os.environ.get(key.lower()) or os.environ.get(key.upper())
+    result = result or default
+    if is_bool:
+        result = strtobool(result)
+    if is_multiline:
+        result = list(filter(None, result.split('\n')))
+    return result
+
+
+def create_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--working-root-directory',
+        action=StoreAction,
+        default=get_env_var(key='remote_home_dir'),
+        type=Path,
+    )
+    parser.add_argument(
+        '--pgp-sign-keyid',
+        action=StoreAction,
+        default=get_env_var(
+            key='pgp_sign_keyid',
+            default='',
+        ),
+        type=str,
+    )
+    parser.add_argument(
+        '--arch',
+        action=StoreAction,
+        default=get_env_var(key='arch'),
+        type=str,
+    )
+    parser.add_argument(
+        '--product-name',
+        action=StoreAction,
+        default=get_env_var(key='product_name'),
+        type=str,
+    )
+    parser.add_argument(
+        '--distribution-major-version',
+        action=StoreAction,
+        default=get_env_var(key='distribution_major_version'),
+        type=int,
+    )
+    parser.add_argument(
+        '--distribution-minor-version',
+        action=StoreAction,
+        default=get_env_var(key='distribution_minor_version'),
+        type=int,
+    )
+    parser.add_argument(
+        '--beta-suffix',
+        action='store',
+        default=get_env_var(key='beta_suffix', default=''),
+        type=str,
+    )
+    parser.add_argument(
+        '--not-needed-variant',
+        action=StoreAction,
+        default=get_env_var(key='not_needed_variant', default='Minimal'),
+        type=str,
+    )
+    parser.add_argument(
+        '--keep-builds',
+        action=StoreAction,
+        type=int,
+        default=get_env_var(key='keep_builds', default=1),
+    )
+    parser.add_argument(
+        '--env-files',
+        action='store',
+        type=List[str],
+        default=get_env_var(
+            key='add_env_files',
+            default='',
+            is_multiline=True,
+        ),
+        nargs='+',
+    )
+    parser.add_argument(
+        '--local-repos',
+        action='store',
+        type=List[str],
+        default=get_env_var(
+            key='local_repos',
+            default='',
+            is_multiline=True,
+        ),
+        nargs='+',
+    )
+    parser.add_argument(
+        '--sigkeys-fingerprints',
+        action=StoreAction,
+        default=get_env_var(
+            key='sigkeys_fingerprints',
+            default='',
+            is_multiline=True,
+        ),
+        nargs='+',
+    )
+    parser.add_argument(
+        '--skip-mirroring',
+        action='store_true',
+        default=get_env_var(
+            key='skip_mirroring',
+            default='False',
+            is_bool=True,
+        ),
+    )
+    parser.add_argument(
+        '--use-products-repos',
+        action='store_true',
+        default=get_env_var(
+            key='use_products_repos',
+            default='False',
+            is_bool=True,
+        ),
+    )
+    parser.add_argument(
+        '--git-auth-token',
+        action='store',
+        type=str,
+        default=get_env_var(key='git_auth_token'),
+    )
+    parser.add_argument(
+        '--git-auth-username',
+        action='store',
+        type=str,
+        default=get_env_var(key='git_auth_username'),
+    )
+    parser.add_argument(
+        '--git-url',
+        action='store',
+        type=str,
+        default=get_env_var(
+            key='git_storage_url',
+            default='git.almalinux.org',
+        ),
+    )
+    parser.add_argument(
+        '--git-project',
+        action='store',
+        type=str,
+        default=get_env_var(
+            key='git_project',
+            default='almalinux/pungi-almalinux',
+        ),
+    )
+    parser.add_argument(
+        '--git-type',
+        action='store',
+        type=str,
+        default=get_env_var(key='git_type', default='gitea'),
+    )
+    parser.add_argument(
+        '--sign-service-username',
+        action='store',
+        help='An username of a sign service',
+        default=get_env_var(
+            key='sign_service_username',
+            default='',
+        ),
+    )
+    parser.add_argument(
+        '--sign-service-password',
+        action='store',
+        help='A password of a sign service',
+        default=get_env_var(
+            key='sign_service_password',
+            default='',
+        ),
+    )
+    parser.add_argument(
+        '--sign-service-endpoint',
+        action='store',
+        help='An endpoint of a sign service',
+        default=get_env_var(
+            key='sign_service_endpoint',
+            default='',
+        ),
+    )
+    parser.add_argument(
+        '-e',
+        '--koji-excluded-packages',
+        required=False,
+        nargs='+',
+        type=List[str],
+        default=get_env_var(
+            key='koji_excluded_packages',
+            default='',
+            is_multiline=True,
+        ),
+    )
+    parser.add_argument(
+        '--branch',
+        action=StoreAction,
+        type=str,
+        default=get_env_var(key='branch', default='master'),
+    )
+    return parser
+
+
+def main():
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    sensitive_arguments = (
+        'git_auth_token',
+        'sign_service_password',
+    )
+    args = create_parser().parse_args()
+    logging.info('All CLI arguments:')
+    for arg_name, arg_value in vars(args).items():
+        if arg_name in sensitive_arguments:
+            continue
+        logging.info('%s: %s', arg_name, arg_value)
+    runner = Runner(
+        working_root_directory=args.working_root_directory,
+        product_name=args.product_name,
+        distribution_major_version=args.distribution_major_version,
+        distribution_minor_version=args.distribution_minor_version,
+        arch=args.arch,
+        branch=args.branch,
+        keep_builds=args.keep_builds,
+        use_products_repos=args.use_products_repos,
+        env_files=args.env_files,
+        beta_suffix=args.beta_suffix,
+        sigkeys_fingerprints=args.sigkeys_fingerprints,
+        skip_mirroring=args.skip_mirroring,
+        local_repos=args.local_repos,
+        not_needed_variant=args.not_needed_variant,
+        pgp_sign_keyid=args.pgp_sign_keyid,
+        git_url=args.git_url,
+        git_project=args.git_project,
+        git_auth_token=args.git_auth_token,
+        git_auth_username=args.git_auth_username,
+        git_type=args.git_type,
+        sign_service_username=args.sign_service_username,
+        sign_service_password=args.sign_service_password,
+        sign_service_endpoint=args.sign_service_endpoint,
+        koji_excluded_packages=args.koji_excluded_packages,
+    )
+    try:
+        runner.checkout_scripts()
+        runner.prepare()
+        runner.build()
+        runner.post()
+    finally:
+        runner.cleanup()
+
+
+if __name__ == '__main__':
+    main()

build_scripts/post_actions.py

@@ -0,0 +1,798 @@
+#!/usr/bin/env python3
+# coding=utf-8
+import json
+import os
+import logging
+from collections import defaultdict
+from shutil import (
+    rmtree,
+    copytree,
+    copy,
+)
+
+import argparse
+import requests
+
+from subprocess import check_call
+from configparser import ConfigParser
+from time import time
+from typing import (
+    List,
+    Optional, 
+    Dict,
+)
+from pathlib import Path
+from urllib.parse import urljoin
+from requests.auth import AuthBase
+
+
+PUNGI_RESULTS = 'pungi-results'
+KICKSTART_REPO = 'BaseOS'
+INCLUDED_TO_KICKSTART_REPO = 'AppStream'
+
+PATH_DICTIONARY = defaultdict(lambda: None, **{
+    'debug': 'debug/{arch}',
+    'source': 'Source',
+    'source_packages': 'Source/Packages',
+    'source_repository': 'Source',
+    'source_tree': 'Source',
+    'debug_packages': 'debug/{arch}/Packages',
+    'debug_repository': 'debug/{arch}',
+    'debug_tree': 'debug/{arch}',
+})  # type: Dict[str ,Optional[str]]
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+class BearerAuth(AuthBase):
+    def __init__(self, token):
+        self.token = token
+
+    def __call__(self, request):
+        request.headers["Authorization"] = f'Bearer {self.token}'
+        return request
+
+
+class Signer:
+    __session__ = None  # type: Optional[requests.Session]
+
+    def get_token(self, username, password) -> str:
+        data = {
+            'email': username,
+            'password': password
+        }
+        response = requests.post(
+            url=urljoin(self.endpoint + '/', 'token'),
+            json=data,
+        )
+        response.raise_for_status()
+        return response.json()['token']
+
+    def __init__(self, username, password, endpoint):
+        self.endpoint = endpoint
+        self.token = self.get_token(
+            username=username,
+            password=password,
+        )
+
+    def sign(
+            self,
+            file_path: Path,
+            keyid: str,
+            sign_type: str = 'detach-sign',
+    ) -> Path:
+        auth = BearerAuth(token=self.token)
+        params = {
+            'keyid': keyid,
+            'sign_type': sign_type,
+        }
+        files = {
+            'file': file_path.open('rb'),
+        }
+        response = requests.post(
+            url=urljoin(self.endpoint + '/', 'sign'),
+            params=params,
+            files=files,
+            auth=auth,
+        )
+        response.raise_for_status()
+        if sign_type == 'detach-sign':
+            file_path = file_path.with_suffix(file_path.suffix + '.asc')
+        with file_path.open('w') as fd:
+            fd.write(response.text)
+        return file_path
+
+    @staticmethod
+    def verify(file_path: Path):
+        if file_path.suffix == '.asc':
+            command = f'gpg --verify {file_path} {file_path.with_suffix("")}'
+        else:
+            command = f'gpg --verify {file_path}'
+        check_call(
+            command,
+            shell=True,
+            universal_newlines=True,
+        )
+
+
+def move_sources_folder_to_right_place(
+        latest_path: Path,
+        repo_name: str,
+):
+    src_of_sources = latest_path.joinpath(repo_name, 'source')
+    dst_of_sources = latest_path.joinpath(repo_name, PATH_DICTIONARY['source'])
+    if src_of_sources.exists():
+        logging.info(
+            'Move sources to right place for result '
+            'dir "%s" and repo "%s"',
+            latest_path,
+            repo_name,
+        )
+        src_of_sources.joinpath('tree').rename(dst_of_sources)
+        rmtree(src_of_sources)
+
+
+def move_debug_folder_to_right_place(
+        latest_path: Path,
+        repo_name: str,
+        arch: str,
+):
+    src_of_debug = latest_path.joinpath(repo_name, arch, 'debug')
+    dst_of_debug = latest_path.joinpath(
+        repo_name,
+        PATH_DICTIONARY['debug'].format(arch=arch),
+    )
+    if src_of_debug.exists():
+        logging.info(
+            'Move a folder with debug rpms to right places for '
+            'result dir "%s", arch "%s" and repo "%s"',
+            latest_path,
+            arch,
+            repo_name,
+        )
+        os.makedirs(dst_of_debug.parent, exist_ok=True)
+        src_of_debug.joinpath('tree').rename(dst_of_debug)
+        rmtree(src_of_debug)
+
+
+def copy_updateinfo_from_platform_repos(
+        src_repos_path: Path,
+        latest_path: Path,
+        repo_name: str,
+        arch: str,
+):
+    dst_path = latest_path.joinpath(repo_name, arch, 'os')
+    if not dst_path.exists():
+        return
+    for path in src_repos_path.glob(
+            f'platform-almalinux-[0-9]-{repo_name.lower()}-'
+            f'{arch}/repodata/*updateinfo*'
+    ):
+        dst_file = dst_path.joinpath('repodata', path.name)
+        if dst_file.exists():
+            return
+        logging.info('Copy updateinfo.xml for repo "%s"', repo_name)
+        copy(
+            path,
+            dst_file,
+        )
+        dst_repodata_path = dst_path.joinpath('repodata')
+        logging.info('Modify repo "%s" with updateinfo.xml', repo_name)
+        check_call(
+            f'modifyrepo_c --mdtype=updateinfo {dst_file} {dst_repodata_path}',
+            shell=True,
+        )
+        return
+    logging.warning(
+        'Updateinfo.xml for repo "%s" does not exist',
+        repo_name,
+    )
+
+
+def sign_repomd_xml(
+        latest_path: Path,
+        repo_name: str,
+        arch: str,
+        pgp_keyid: str,
+        username: str,
+        password: str,
+        endpoint: str,
+):
+    repomd_xml_path_suffix = 'repodata/repomd.xml'
+    os_repomd_xml_path = latest_path.joinpath(
+        repo_name,
+        arch,
+        'os',
+        repomd_xml_path_suffix,
+    )
+    kickstart_repomd_xml_path = latest_path.joinpath(
+        repo_name,
+        arch,
+        'kickstart',
+        repomd_xml_path_suffix,
+    )
+    source_repomd_xml_path = latest_path.joinpath(
+        repo_name,
+        PATH_DICTIONARY['source'],
+        repomd_xml_path_suffix,
+    )
+    debug_repomd_xml_path = latest_path.joinpath(
+        repo_name,
+        PATH_DICTIONARY['debug'].format(arch=arch),
+        repomd_xml_path_suffix
+    )
+    logging.info(
+        'Sign repomd.xml files for "%s" and verify signatures',
+        repo_name,
+    )
+    for repomd_xml_path in (
+        os_repomd_xml_path,
+        kickstart_repomd_xml_path,
+        source_repomd_xml_path,
+        debug_repomd_xml_path,
+    ):
+        if not repomd_xml_path.exists():
+            continue
+        signer = Signer(
+            username=username,
+            password=password,
+            endpoint=endpoint,
+        )
+        file_path = signer.sign(
+            file_path=repomd_xml_path,
+            keyid=pgp_keyid,
+        )
+        signer.verify(file_path=file_path)
+
+
+def create_kickstart_folder(
+        latest_path: Path,
+        repo_name: str,
+        arch: str,
+):
+    src_kickstart = latest_path.joinpath(repo_name, arch, 'os')
+    dst_kickstart = latest_path.joinpath(repo_name, arch, 'kickstart')
+    if src_kickstart.exists() and not dst_kickstart.exists():
+        logging.info(
+            'Make kickstart repo for result dir "%s", '
+            'repo "%s" and arch "%s"',
+            latest_path,
+            repo_name,
+            arch,
+        )
+        copytree(
+            src_kickstart,
+            dst_kickstart,
+            copy_function=os.link,
+        )
+        logging.info(
+            'Copy repodata for a kickstart without using hardlinks'
+        )
+        repodata_dst_path = dst_kickstart.joinpath('repodata')
+        repodata_src_path = src_kickstart.joinpath('repodata')
+        rmtree(repodata_dst_path)
+        copytree(
+            repodata_src_path,
+            repodata_dst_path,
+        )
+
+
+def update_timestamp_in_treeinfo(
+        tree_info_path: Path,
+        timestamp: int,
+):
+    replaced_values = {
+        'general': 'timestamp',
+        'tree': 'build_timestamp',
+    }
+    if not tree_info_path.exists():
+        return
+    tree_info_config = ConfigParser()
+    tree_info_config.read(tree_info_path)
+    for section, key in replaced_values.items():
+        tree_info_config.set(
+            section=section,
+            option=key,
+            value=str(timestamp),
+        )
+    logging.info(
+        'Update timestamp "%s" in .treeinfo for "%s"',
+        timestamp,
+        tree_info_path,
+    )
+    with open(tree_info_path, 'w') as tree_info_fp:
+        tree_info_config.write(tree_info_fp)
+
+
+def update_kickstart_treeinfo_file(
+        tree_info_path: str,
+):
+    replaced_values = {
+        'packages': None,
+        'repository': None,
+    }
+    tree_info_config = ConfigParser()
+    tree_info_config.read(tree_info_path)
+    logging.info(
+        'Update .treeinfo file "%s": replace path-suffix `os` by `kickstart`',
+        tree_info_path,
+    )
+    section_name = f'variant-{INCLUDED_TO_KICKSTART_REPO}'
+    for key in replaced_values:
+        if section_name not in tree_info_config.sections():
+            continue
+        replaced_values[key] = tree_info_config.get(
+            section=section_name,
+            option=key
+        ).replace('os', 'kickstart')
+    for key, value in replaced_values.items():
+        if section_name not in tree_info_config.sections():
+            continue
+        tree_info_config.set(
+            section=section_name,
+            option=key,
+            value=value,
+        )
+    # because it's hardlink and could be modified
+    # both files (in dirs `os` and `kickstart`)
+    os.remove(tree_info_path)
+
+    with open(tree_info_path, 'w') as tree_info_fp:
+        tree_info_config.write(tree_info_fp)
+
+
+def sign_isos_checksum(
+        latest_path: Path,
+        arch: str,
+        pgp_keyid: str,
+        username: str,
+        password: str,
+        endpoint: str,
+):
+    logging.info('Sign ISOs CHECKSUM and verify signature')
+    checksum_path = latest_path.joinpath('isos', arch, 'CHECKSUM')
+    if not checksum_path.exists():
+        logging.warning('File CHECKSUM is absent')
+        return
+    signer = Signer(
+        username=username,
+        password=password,
+        endpoint=endpoint,
+    )
+    file_path = signer.sign(
+        file_path=checksum_path,
+        keyid=pgp_keyid,
+        sign_type='clear-sign',
+    )
+    signer.verify(file_path=file_path)
+
+
+def post_processing_images_json_metadata(
+        latest_path: Path,
+        arch: str,
+):
+    images_json_path = latest_path.joinpath(
+        'metadata', arch, 'images.json'
+    )
+    logging.info('Post-processing images.json')
+    if not images_json_path.exists():
+        logging.warning('images.json is absent')
+        return
+    with open(images_json_path, 'r') as images_metadata_fd:
+        content = json.load(images_metadata_fd)
+        images = content['payload']['images']
+        for variant in images:
+            variant_data = images[variant]  # type: Dict[str, List[Dict]]
+            for arch, images_list in variant_data.items():
+                variant_data[arch] = [
+                    dict(
+                        image,
+                        **{
+                            'path': str(Path('isos').joinpath(
+                                arch,
+                                Path(image['path']).name,
+                            ))
+                        }
+                    ) for image in images_list
+                ]
+
+    with open(images_json_path, 'w') as images_metadata_fd:
+        json.dump(
+            content,
+            images_metadata_fd,
+            indent=4,
+        )
+
+
+def post_processing_rpms_json_metadata(
+        latest_path: Path,
+        arch: str,
+):
+    rpms_json_path = latest_path.joinpath(
+        'metadata', arch, 'rpms.json'
+    )
+    logging.info('Post-processing rpms.json')
+    if not rpms_json_path.exists():
+        logging.warning('rpms.json is absent')
+        return
+    with open(rpms_json_path, 'r') as rpms_metadata_fd:
+        content = json.load(rpms_metadata_fd)
+        rpms = content['payload']['rpms']
+        for variant in rpms:
+            variant_data = rpms[variant]
+            if variant == 'Minimal':
+                continue
+            for arch in variant_data:
+                arch_data = variant_data[arch]
+                for srpm in arch_data:
+                    srpm_data = arch_data[srpm]
+                    for artifact in srpm_data:
+                        artifact_data = srpm_data[artifact]
+                        path_suffix = PATH_DICTIONARY[
+                            artifact_data['category']
+                        ]  # type: Optional[str]
+                        if path_suffix is None:
+                            continue
+                        else:
+                            path_suffix = path_suffix.format(arch=arch)
+                        artifact_path = Path(artifact_data['path'])
+                        artifact_data['path'] = str(Path(variant).joinpath(
+                            path_suffix,
+                            artifact_path.parent.name,
+                            artifact_path.name,
+                        ))
+
+    with open(rpms_json_path, 'w') as rpms_metadata_fd:
+        if 'Minimal' in content['payload']['rpms']:
+            del content['payload']['rpms']['Minimal']
+        json.dump(
+            content,
+            rpms_metadata_fd,
+            indent=4,
+        )
+
+
+def post_processing_compose_info_json_metadata(
+        latest_path: Path,
+        arch: str,
+):
+    composeinfo_json_path = latest_path.joinpath(
+        'metadata', arch, 'composeinfo.json'
+    )
+
+    logging.info('Post-processing composeinfo.json')
+    if not composeinfo_json_path.exists():
+        logging.warning('composeinfo.json is absent')
+        return
+    with open(composeinfo_json_path, 'r') as composeinfo_metadata_fd:
+        content = json.load(composeinfo_metadata_fd)
+        variants = content['payload']['variants']
+        for variant in variants:
+            variant_paths = variants[variant]['paths']
+            if variant == 'Minimal':
+                continue
+            for path_type in variant_paths:
+                path_data = variant_paths[path_type]  # type: Dict
+                for arch, path in path_data.items():
+                    path_suffix = PATH_DICTIONARY[
+                        path_type]  # type: Optional[str]
+                    if path_suffix is None:
+                        continue
+                    else:
+                        path_suffix = path_suffix.format(arch=arch)
+                    path = Path(variant).joinpath(path_suffix)
+                    path_data[arch] = str(path)
+
+    with open(composeinfo_json_path, 'w') as composeinfo_metadata_fd:
+        if 'Minimal' in content['payload']['variants']:
+            del content['payload']['variants']['Minimal']
+        json.dump(
+            content,
+            composeinfo_metadata_fd,
+            indent=4,
+        )
+
+
+def move_json_metadata_to_arch_folder(
+        latest_path: Path,
+        arch: str,
+):
+    extension = '*.json'
+    metadata_path = latest_path.joinpath('metadata')
+    metadata_arch_path = metadata_path.joinpath(arch)
+    os.makedirs(metadata_arch_path, exist_ok=True)
+    for json_metadata in Path(metadata_path).glob(extension):
+        logging.info(
+            'Copy "%s" to arch directory "%s"',
+            json_metadata,
+            metadata_arch_path,
+        )
+        json_metadata.rename(metadata_arch_path.joinpath(json_metadata.name))
+
+
+def move_iso_and_its_artifacts_to_isos_arch_folder(
+        src_latest_path: Path,
+        repo_name: str,
+        arch: str,
+        exts_of_files: List[str],
+        dst_latest_path: Optional[Path] = None,
+):
+    src_iso_folder_path = src_latest_path.joinpath(repo_name, arch, 'iso')
+    isos_arch_folder = (dst_latest_path or src_latest_path).joinpath(
+        'isos',
+        arch,
+    )
+    os.makedirs(isos_arch_folder, exist_ok=True)
+    if not src_iso_folder_path.exists():
+        return
+    for ext_of_file in exts_of_files:
+        for src_file_path in src_iso_folder_path.glob(ext_of_file):
+            dst_file_path = isos_arch_folder.joinpath(src_file_path.name)
+            if not src_file_path.exists():
+                continue
+            if dst_file_path.exists():
+                continue
+            logging.info(
+                'Move iso or iso\'s artifacts from "%s" to "%s"',
+                src_file_path,
+                dst_file_path,
+            )
+            src_file_path.rename(dst_file_path)
+    src_checksum_path = src_latest_path.joinpath(
+        repo_name,
+        arch,
+        'iso',
+        'CHECKSUM',
+    )
+    dst_checksum_path = isos_arch_folder.joinpath('CHECKSUM')
+    if src_checksum_path.exists():
+        logging.info(
+            'Write CHEKSUM from "%s" to "%s"',
+            src_checksum_path,
+            dst_checksum_path,
+        )
+        with open(src_checksum_path, 'r') as src_checksum_file, \
+             open(dst_checksum_path, 'a+') as dst_checksum_path:
+            src_checksum_content = src_checksum_file.read()
+            dst_checksum_path.write(src_checksum_content)
+    rmtree(src_iso_folder_path)
+
+
+def rename_latest_dir(
+        latest_path: Path,
+):
+    old_real_name = Path(os.readlink(latest_path.parent)).name
+    if not old_real_name.startswith('last_compose_dir'):
+        return
+    old_real_name_path = latest_path.parent.parent.joinpath(old_real_name)
+    new_real_name = f'{int(time())}-{old_real_name}'
+    new_real_name_path = latest_path.parent.parent.joinpath(new_real_name)
+    logging.info('New real name path %s', new_real_name_path)
+    logging.info('Old real name path %s', old_real_name_path)
+    if not old_real_name_path.exists():
+        return
+    logging.info(
+        'Add the timestamp to name of a '
+        'real directory with latest result: "%s"',
+        new_real_name_path,
+    )
+    old_real_name_path.rename(new_real_name_path)
+    os.unlink(latest_path.parent)
+    os.symlink(new_real_name_path, latest_path.parent)
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--env-path',
+        action='store',
+        help='A path to folder which will be used '
+             'for building new distribution',
+        required=True,
+    )
+    parser.add_argument(
+        '--sign-service-username',
+        action='store',
+        help='An username of a sign service',
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        '--sign-service-password',
+        action='store',
+        help='A password of a sign service',
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        '--sign-service-endpoint',
+        action='store',
+        help='An endpoint of a sign service',
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        '--pgp-sign-keyid',
+        action='store',
+        help='PGP sign key ID. Used for signing building artifacts',
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '--middle-result-directory',
+        action='store',
+        help='A directory with middle result. '
+             'E.g. a directory contains Minimal iso',
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        '--arch',
+        type=str,
+        help='Architecture of a distribution',
+        required=True,
+    )
+    parser.add_argument(
+        '--source-repos-folder',
+        type=str,
+        help='Path to folder there are stored source repos',
+        required=True,
+    )
+    parser.add_argument(
+        '--repos',
+        nargs='+',
+        type=str,
+        help='A list of repositories are contained in distribution',
+        required=True,
+    )
+    parser.add_argument(
+        '--middle-repos',
+        nargs='+',
+        type=str,
+        help='A list of repositories from a middle result '
+             'which will be used for getting ISOs',
+        required=False,
+        default=[],
+    )
+    parser.add_argument(
+        '--not-needed-repos',
+        nargs='*',
+        type=str,
+        help='A list of repositories which are not needed, e.g. Minimal',
+        required=False,
+        default=[],
+    )
+
+    return parser
+
+
+def cli_main():
+
+    args = create_parser().parse_args()
+    pungi_results = Path(args.env_path).joinpath(PUNGI_RESULTS)
+    latest_result_paths = pungi_results.glob('latest-*')
+    logging.info(
+        'We have the following latest results "%s"',
+        latest_result_paths,
+    )
+    extensions_of_files = ['*.iso', '*.manifest']
+    for latest_path in latest_result_paths:
+        latest_path = latest_path.joinpath('compose')
+        build_timestamp = int(time())
+
+        for repo in args.repos:
+            if not latest_path.joinpath(repo).exists():
+                continue
+            if repo in args.middle_repos:
+                not_needed_repo_path = latest_path.joinpath(repo)
+                if not_needed_repo_path.exists():
+                    rmtree(not_needed_repo_path)
+                continue
+            move_sources_folder_to_right_place(
+                latest_path=latest_path,
+                repo_name=repo,
+            )
+            move_debug_folder_to_right_place(
+                latest_path=latest_path,
+                repo_name=repo,
+                arch=args.arch,
+            )
+            copy_updateinfo_from_platform_repos(
+                src_repos_path=Path(args.source_repos_folder),
+                latest_path=latest_path,
+                repo_name=repo,
+                arch=args.arch,
+            )
+            create_kickstart_folder(
+                latest_path=latest_path,
+                repo_name=repo,
+                arch=args.arch,
+            )
+            if all(opt is not None for opt in (
+                args.sign_service_username,
+                args.sign_service_password,
+                args.sign_service_endpoint
+            )):
+                sign_repomd_xml(
+                    latest_path=latest_path,
+                    repo_name=repo,
+                    arch=args.arch,
+                    username=args.sign_service_username,
+                    password=args.sign_service_password,
+                    endpoint=args.sign_service_endpoint,
+                    pgp_keyid=args.pgp_sign_keyid,
+                )
+            move_iso_and_its_artifacts_to_isos_arch_folder(
+                src_latest_path=latest_path,
+                repo_name=repo,
+                arch=args.arch,
+                exts_of_files=extensions_of_files,
+            )
+            repo_path = latest_path.joinpath(repo)
+            for path in Path(repo_path).rglob('.treeinfo'):
+                update_timestamp_in_treeinfo(
+                    tree_info_path=path,
+                    timestamp=build_timestamp,
+                )
+                if path.parent.name == 'kickstart':
+                    update_kickstart_treeinfo_file(
+                        tree_info_path=str(path),
+                    )
+        move_json_metadata_to_arch_folder(
+            latest_path=latest_path,
+            arch=args.arch,
+        )
+        post_processing_compose_info_json_metadata(
+            latest_path=latest_path,
+            arch=args.arch,
+        )
+        post_processing_rpms_json_metadata(
+            latest_path=latest_path,
+            arch=args.arch,
+        )
+        post_processing_images_json_metadata(
+            latest_path=latest_path,
+            arch=args.arch,
+        )
+        if all(opt is not None for opt in (
+            args.sign_service_username,
+            args.sign_service_password,
+            args.sign_service_endpoint
+        )):
+            sign_isos_checksum(
+                latest_path=latest_path,
+                arch=args.arch,
+                username=args.sign_service_username,
+                password=args.sign_service_password,
+                endpoint=args.sign_service_endpoint,
+                pgp_keyid=args.pgp_sign_keyid,
+            )
+        for repo in args.middle_repos:
+            if args.middle_result_directory is not None:
+                move_iso_and_its_artifacts_to_isos_arch_folder(
+                    src_latest_path=Path(
+                        args.middle_result_directory,
+                    ).joinpath('compose'),
+                    repo_name=repo,
+                    arch=args.arch,
+                    exts_of_files=extensions_of_files,
+                    dst_latest_path=latest_path,
+                )
+        for repo in args.not_needed_repos:
+            not_needed_repo_path = latest_path.joinpath(repo)
+            if not_needed_repo_path.exists():
+                logging.info(
+                    'Remove not needed variant "%s" by path "%s"',
+                    repo,
+                    not_needed_repo_path,
+                )
+                rmtree(not_needed_repo_path)
+        rename_latest_dir(latest_path=latest_path)
+    if args.middle_result_directory is not None and \
+            Path(args.middle_result_directory).exists():
+        rmtree(args.middle_result_directory)
+
+
+if __name__ == '__main__':
+    cli_main()

build_scripts/prepare.py

@@ -0,0 +1,537 @@
+#!/usr/bin/env python3
+# coding=utf-8
+import argparse
+import base64
+import json
+import logging
+import os
+import subprocess
+from configparser import ConfigParser
+from pathlib import Path
+from typing import (
+    List,
+    Dict, Optional,
+)
+
+import jinja2
+import requests
+
+from urllib.parse import quote, quote_plus
+from requests.auth import HTTPBasicAuth
+
+VARIANTS_GENERATOR_FOLDER = 'variants-xml-generator'
+COMPS_GENERATOR_FOLDER = 'almacomps'
+KOJI_CONF_PATH = '/etc/koji.conf'
+
+VARIANTS_OPTIONS_FILENAME = 'variants_options.json'
+PUNGI_BUILD_CONF_TEMPLATE_FILENAME = 'pungi-build.conf.j2'
+PUNGI_BUILD_CONF_FILENAME = 'pungi-build.conf'
+INCLUDE_EXCLUDE_CONF_FILENAME = 'include_exclude.conf'
+MULTILIB_CONF_FILENAME = 'multilib.conf'
+EXTRA_OPTIONS_CONF_FILENAME = 'extra_options.conf'
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def dnf_reposync_mirroring(
+        use_products_repos: bool,
+        product_name: str,
+        distribution_major_version: str,
+        arch: str,
+        mirroring_target: str
+) -> None:
+    base_repos_dir = Path('/etc/yum.repos.d/').joinpath(
+        product_name.lower(),
+        distribution_major_version,
+        arch,
+    )
+    specific_repos = base_repos_dir.joinpath('specific_repos')
+    products_repos = base_repos_dir.joinpath('products_repos')
+    platform_repos = base_repos_dir.joinpath('platform_repos')
+    if use_products_repos and any(products_repos.iterdir()):
+        command = f'dnf reposync -p {mirroring_target}-products_repos ' \
+                  f'--setopt=reposdir="{products_repos}" ' \
+                  '--enablerepo=* --download-metadata ' \
+                  '--delete --downloadcomps --remote-time '
+        subprocess.check_call(
+            command,
+            shell=True,
+        )
+    if any(specific_repos.iterdir()):
+        command = f'dnf reposync -p {mirroring_target} ' \
+                  f'--setopt=reposdir="{specific_repos}" ' \
+                  '--enablerepo=* --download-metadata ' \
+                  '--delete --downloadcomps --remote-time '
+        subprocess.check_call(
+            command,
+            shell=True,
+        )
+    if any(platform_repos.iterdir()):
+        command = f'dnf reposync -p {mirroring_target} ' \
+                  f'--setopt=reposdir="{platform_repos}" ' \
+                  '--enablerepo=* --download-metadata ' \
+                  '--delete --downloadcomps --remote-time '
+        subprocess.check_call(
+            command,
+            shell=True,
+        )
+
+
+def load_remote_file_content(
+        name: str,
+        distribution_major_version: int,
+        distribution_minor_version: int,
+        beta_suffix: str,
+        product_name: str,
+        git_auth_token: str,
+        git_auth_username: str,
+        git_url: str,
+        git_project: str,
+        git_type: str = 'gitea',
+
+) -> Optional[str]:
+    beta_suffix = beta_suffix if beta_suffix else ''
+    decoding_func = {
+        'gerrit': lambda content: base64.b64decode(content).decode('utf-8'),
+        'gitea': lambda content: base64.b64decode(json.loads(content.decode())
+                                                  ['content']).decode('utf-8'),
+        'github': lambda content: base64.b64decode(content).decode('utf-8'),
+    }
+    if 'gerrit' == git_type:
+        name = quote(name, safe='')
+        url = f'https://{git_url}/a/projects/{git_project}/branches/' \
+              f'{product_name[0].lower()}{distribution_major_version}.' \
+              f'{distribution_minor_version}' \
+              f'{beta_suffix}/files/{name}/content'
+        response = requests.get(url, auth=HTTPBasicAuth(
+            username=git_auth_username,
+            password=git_auth_token,
+        ))
+    elif 'gitea' == git_type:
+        headers = {
+            'accept': 'application/json',
+        }
+        params = {
+            'access_token': git_auth_token,
+            'ref': (
+                f'{product_name[0].lower()}{distribution_major_version}.'
+                f'{distribution_minor_version}{beta_suffix}'
+            )
+        }
+        name = quote(name, safe='')
+        url = f'https://{git_url}/api/v1/repos/{git_project}/contents/{name}'
+        response = requests.get(url, params=params, headers=headers)
+    elif 'github' == git_type:
+        name = quote(name, safe='')
+        headers = {
+            'Authorization:': f'Bearer {git_auth_token}',
+            'Accept': 'application/vnd.github+json',
+        }
+        params = {
+            'ref': (
+                f'{product_name[0].lower()}{distribution_major_version}.'
+                f'{distribution_minor_version}{beta_suffix}'
+            )
+        }
+        url = f'https://api.github/repos/{git_project}/contents/{name}'
+        response = requests.get(url, params=params, headers=headers)
+    else:
+        raise NotImplemented(f'{git_type} is not supported yet')
+    try:
+        response.raise_for_status()
+    except requests.RequestException:
+        return
+    return decoding_func[git_type](response.content)
+
+
+def render_variants_options(
+        env: jinja2.Environment,
+        variables: Dict,
+) -> dict:
+    variants_options_template = env.get_template(os.path.join(
+        variables['arch'],
+        VARIANTS_OPTIONS_FILENAME,
+    ))
+    variants_options = variants_options_template.render(**variables)
+    return json.loads(variants_options)
+
+
+def render_pungi_build_conf(
+        env: jinja2.Environment,
+        variables: Dict,
+) -> str:
+    pungi_build_conf_template = env.get_template(
+        PUNGI_BUILD_CONF_TEMPLATE_FILENAME,
+    )
+    return pungi_build_conf_template.render(**variables)
+
+
+def prepare_build_conf(
+        product_name: str,
+        arch: str,
+        distribution_major_version: int,
+        distribution_minor_version: int,
+        env_path: str,
+        sigkeys_fingerprints: List[str],
+        git_url: str,
+        git_project: str,
+        git_auth_token: str,
+        git_auth_username: str,
+        git_type: str = 'gitea',
+        beta_suffix: str = '',
+):
+    logging.info(
+        'Prepare build conf'
+    )
+    variables = {
+        'product_name': product_name,
+        'arch': arch,
+        'distribution_major_version': distribution_major_version,
+        'distribution_minor_version': distribution_minor_version,
+        'beta_suffix': beta_suffix,
+        'sigkeys_fingerprints': [
+            f'"{sigkey}"' for sigkey in sigkeys_fingerprints if sigkey
+        ],
+        'git_auth_username': quote_plus(git_auth_username),
+        'git_auth_token': quote_plus(git_auth_token),
+    }
+    env = jinja2.Environment(
+        loader=jinja2.FunctionLoader(lambda name: load_remote_file_content(
+            name=name,
+            distribution_major_version=distribution_major_version,
+            distribution_minor_version=distribution_minor_version,
+            beta_suffix=beta_suffix,
+            git_url=git_url,
+            git_project=git_project,
+            git_auth_token=git_auth_token,
+            git_auth_username=git_auth_username,
+            product_name=product_name,
+            git_type=git_type,
+        )),
+        autoescape=jinja2.select_autoescape(),
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+    variables['variants'] = render_variants_options(
+        env=env,
+        variables=variables,
+    )
+    extra_options_conf_lines = load_remote_file_content(
+        name=f'{arch}/{EXTRA_OPTIONS_CONF_FILENAME}',
+        distribution_minor_version=distribution_minor_version,
+        distribution_major_version=distribution_major_version,
+        beta_suffix=beta_suffix,
+        git_url=git_url,
+        git_project=git_project,
+        git_auth_token=git_auth_token,
+        git_auth_username=git_auth_username,
+        product_name=product_name,
+        git_type=git_type,
+    )
+    variables['extra_options'] = extra_options_conf_lines is not None
+    pungi_build_conf_lines = render_pungi_build_conf(
+        env=env,
+        variables=variables,
+    )
+    include_exclude_conf_lines = load_remote_file_content(
+        name=f'{arch}/{INCLUDE_EXCLUDE_CONF_FILENAME}',
+        distribution_minor_version=distribution_minor_version,
+        distribution_major_version=distribution_major_version,
+        beta_suffix=beta_suffix,
+        git_url=git_url,
+        git_project=git_project,
+        git_auth_token=git_auth_token,
+        git_auth_username=git_auth_username,
+        product_name=product_name,
+        git_type=git_type,
+    )
+    multilib_conf_lines = load_remote_file_content(
+        name=f'{arch}/{MULTILIB_CONF_FILENAME}',
+        distribution_minor_version=distribution_minor_version,
+        distribution_major_version=distribution_major_version,
+        beta_suffix=beta_suffix,
+        git_url=git_url,
+        git_project=git_project,
+        git_auth_token=git_auth_token,
+        git_auth_username=git_auth_username,
+        product_name=product_name,
+        git_type=git_type,
+    )
+    with open(os.path.join(
+            env_path,
+            PUNGI_BUILD_CONF_FILENAME,
+    ), 'w') as fd:
+        fd.write(pungi_build_conf_lines)
+    with open(os.path.join(
+            env_path,
+            INCLUDE_EXCLUDE_CONF_FILENAME,
+    ), 'w') as fd:
+        fd.write(include_exclude_conf_lines)
+    with open(os.path.join(
+            env_path,
+            MULTILIB_CONF_FILENAME,
+    ), 'w') as fd:
+        fd.write(multilib_conf_lines)
+    if variables['extra_options']:
+        with open(os.path.join(
+                env_path,
+                EXTRA_OPTIONS_CONF_FILENAME,
+        ), 'w') as fd:
+            fd.write(extra_options_conf_lines)
+
+
+def add_koji_profile(
+        env_path: str,
+        profile_name: str,
+):
+    logging.info(
+        'Add koji profile "%s" to "%s"',
+        profile_name,
+        KOJI_CONF_PATH,
+    )
+    koji_env_path = os.path.join(
+        env_path,
+        'koji'
+    )
+    with open(KOJI_CONF_PATH, 'r') as koji_conf_file:
+        koji_conf_obj = ConfigParser()
+        koji_conf_obj.read_file(koji_conf_file)
+    if profile_name not in koji_conf_obj.sections():
+        koji_conf_obj.add_section(profile_name)
+    koji_conf_obj.set(profile_name, 'topdir', koji_env_path)
+    with open(KOJI_CONF_PATH, 'w') as koji_conf_file:
+        koji_conf_obj.write(koji_conf_file)
+
+
+def save_additional_env_files(
+        env_path: str,
+        add_env_files: List[str],
+) -> None:
+    for add_env_file in filter(lambda i: True if i else False, add_env_files):
+        env_file_name, file_content_in_base64 = add_env_file.split(',')
+        env_file_content = base64.b64decode(
+            file_content_in_base64,
+        ).decode('utf-8')
+        env_file_path = os.path.join(
+            env_path,
+            env_file_name,
+        )
+        with open(env_file_path, 'w') as env_file:
+            env_file.write(env_file_content)
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--env-path',
+        action='store',
+        help='A path to folder which will be used '
+             'for building new distribution',
+        required=False,
+        default=None,
+    )
+
+    subparsers = parser.add_subparsers(
+        dest='command',
+    )
+
+    parser_synchronize_using_dnf_reposync = subparsers.add_parser(
+        'dnf_reposync_synchronize',
+        help='Run synchronize using DNF reposync',
+    )
+
+    parser_synchronize_using_dnf_reposync.add_argument(
+        '--mirroring-dnf-repos',
+        nargs='*',
+        type=str,
+        help='A list of repos which will be mirrored',
+        required=False,
+    )
+    parser_synchronize_using_dnf_reposync.add_argument(
+        '--mirroring-target',
+        action='store',
+        help='A folder which will contain a local mirror',
+        required=False,
+    )
+    parser_synchronize_using_dnf_reposync.add_argument(
+        '--use-products-repos',
+        action='store_true',
+        default=False,
+    )
+    parser_synchronize_using_dnf_reposync.add_argument(
+        '--product-name',
+        action='store',
+        help='A name of building product',
+        required=True,
+    )
+    parser_synchronize_using_dnf_reposync.add_argument(
+        '--arch',
+        help='Architecture of a product',
+        action='store',
+        required=True,
+    )
+    parser_synchronize_using_dnf_reposync.add_argument(
+        '--distribution-major-version',
+        help='Major version of a product',
+        action='store',
+        required=True,
+    )
+
+    parser_add_env_files = subparsers.add_parser(
+        'add_env_files',
+        help='Save environment files which are passed as base64 strings'
+    )
+    parser_add_env_files.add_argument(
+        '--env-files',
+        nargs='*',
+        type=str,
+        help='A list of files which should be stored in env '
+             'directory. E.t. `add-comps.xml,<content_in_base64>`',
+        required=False,
+        default=[],
+    )
+
+    parser_koji_profile = subparsers.add_parser(
+        'add_koji_profile',
+        help=f'Add new koji profile to {KOJI_CONF_PATH}',
+    )
+    parser_koji_profile.add_argument(
+        '--koji-profile-name',
+        action='store',
+        help='A name of koji profile',
+        required=True,
+    )
+
+    parser_build_conf = subparsers.add_parser(
+        'prepare_build_conf',
+        help='Prepare a Pungi build conf',
+    )
+    parser_build_conf.add_argument(
+        '--product-name',
+        action='store',
+        help='A name of building product',
+        required=True,
+    )
+    parser_build_conf.add_argument(
+        '--arch',
+        help='Architecture of a product',
+        action='store',
+        required=True,
+    )
+    parser_build_conf.add_argument(
+        '--distribution-major-version',
+        help='Major version of a product',
+        action='store',
+        required=True,
+    )
+    parser_build_conf.add_argument(
+        '--distribution-minor-version',
+        help='Minor version of a product',
+        action='store',
+        required=True,
+    )
+    parser_build_conf.add_argument(
+        '--beta-suffix',
+        help='Suffix of a ISOs & Volume ID names. E.g. `-beta-1`',
+        action='store',
+        default='',
+        type=str,
+        required=False,
+    )
+    parser_build_conf.add_argument(
+        '--sigkeys-fingerprints',
+        nargs='*',
+        type=str,
+        help='A list of fingerprints of AlmaLinux sign keys. '
+             'They are used for checking that all packages are signed',
+    )
+    parser_build_conf.add_argument(
+        '--git-auth-token',
+        action='store',
+        type=str,
+        help='Auth token for access to a Git repository which '
+             'contains a build config and related stuff'
+    )
+    parser_build_conf.add_argument(
+        '--git-auth-username',
+        action='store',
+        type=str,
+        help='Auth username for access to a Git repository which '
+             'contains a build config and related stuff'
+    )
+    parser_build_conf.add_argument(
+        '--git-url',
+        action='store',
+        type=str,
+        help='Git URL for a Git repository which '
+             'contains a build config and related stuff'
+    )
+    parser_build_conf.add_argument(
+        '--git-project',
+        action='store',
+        type=str,
+        help='Name of a Git repository which '
+             'contains a build config and related stuff'
+    )
+    parser_build_conf.add_argument(
+        '--git-type',
+        action='store',
+        type=str,
+        default='gitea',
+        help='Type of a Git repository which '
+             'contains a build config and related stuff'
+    )
+    return parser
+
+
+def check_is_root():
+    if os.geteuid():
+        logging.error('The script should be ran under root or any sudo user')
+        exit(1)
+
+
+def cli_main():
+    # check_is_root()
+    parser = create_parser()
+    args = parser.parse_args()
+    if args.command == 'dnf_reposync_synchronize':
+        dnf_reposync_mirroring(
+            use_products_repos=args.use_products_repos,
+            product_name=args.product_name,
+            distribution_major_version=args.distribution_major_version,
+            arch=args.arch,
+            mirroring_target=args.mirroring_target,
+        )
+    else:
+        # another commands which require an env path
+        os.makedirs(args.env_path, exist_ok=True)
+    if args.command == 'add_env_files':
+        save_additional_env_files(
+            env_path=args.env_path,
+            add_env_files=args.env_files,
+        )
+    if args.command == 'add_koji_profile':
+        add_koji_profile(
+            env_path=args.env_path,
+            profile_name=args.koji_profile_name,
+        )
+    if args.command == 'prepare_build_conf':
+        prepare_build_conf(
+            beta_suffix=args.beta_suffix,
+            product_name=args.product_name,
+            arch=args.arch,
+            distribution_major_version=args.distribution_major_version,
+            distribution_minor_version=args.distribution_minor_version,
+            env_path=args.env_path,
+            sigkeys_fingerprints=args.sigkeys_fingerprints,
+            git_url=args.git_url,
+            git_project=args.git_project,
+            git_auth_token=args.git_auth_token,
+            git_auth_username=args.git_auth_username,
+            git_type=args.git_type,
+        )
+
+
+if __name__ == '__main__':
+    cli_main()