smeserver-pungi/build_scripts/prepare.py
2024-07-05 09:44:42 +10:00

538 lines
17 KiB
Python
Executable File

#!/usr/bin/env python3
# coding=utf-8
import argparse
import base64
import json
import logging
import os
import subprocess
from configparser import ConfigParser
from pathlib import Path
from typing import (
List,
Dict, Optional,
)
import jinja2
import requests
from urllib.parse import quote, quote_plus
from requests.auth import HTTPBasicAuth
VARIANTS_GENERATOR_FOLDER = 'variants-xml-generator'
COMPS_GENERATOR_FOLDER = 'almacomps'
KOJI_CONF_PATH = '/etc/koji.conf'
VARIANTS_OPTIONS_FILENAME = 'variants_options.json'
PUNGI_BUILD_CONF_TEMPLATE_FILENAME = 'pungi-build.conf.j2'
PUNGI_BUILD_CONF_FILENAME = 'pungi-build.conf'
INCLUDE_EXCLUDE_CONF_FILENAME = 'include_exclude.conf'
MULTILIB_CONF_FILENAME = 'multilib.conf'
EXTRA_OPTIONS_CONF_FILENAME = 'extra_options.conf'
logging.basicConfig(level=logging.INFO)
def dnf_reposync_mirroring(
use_products_repos: bool,
product_name: str,
distribution_major_version: str,
arch: str,
mirroring_target: str
) -> None:
base_repos_dir = Path('/etc/yum.repos.d/').joinpath(
product_name.lower(),
distribution_major_version,
arch,
)
specific_repos = base_repos_dir.joinpath('specific_repos')
products_repos = base_repos_dir.joinpath('products_repos')
platform_repos = base_repos_dir.joinpath('platform_repos')
if use_products_repos and any(products_repos.iterdir()):
command = f'dnf reposync -p {mirroring_target}-products_repos ' \
f'--setopt=reposdir="{products_repos}" ' \
'--enablerepo=* --download-metadata ' \
'--delete --downloadcomps --remote-time '
subprocess.check_call(
command,
shell=True,
)
if any(specific_repos.iterdir()):
command = f'dnf reposync -p {mirroring_target} ' \
f'--setopt=reposdir="{specific_repos}" ' \
'--enablerepo=* --download-metadata ' \
'--delete --downloadcomps --remote-time '
subprocess.check_call(
command,
shell=True,
)
if any(platform_repos.iterdir()):
command = f'dnf reposync -p {mirroring_target} ' \
f'--setopt=reposdir="{platform_repos}" ' \
'--enablerepo=* --download-metadata ' \
'--delete --downloadcomps --remote-time '
subprocess.check_call(
command,
shell=True,
)
def load_remote_file_content(
name: str,
distribution_major_version: int,
distribution_minor_version: int,
beta_suffix: str,
product_name: str,
git_auth_token: str,
git_auth_username: str,
git_url: str,
git_project: str,
git_type: str = 'gitea',
) -> Optional[str]:
beta_suffix = beta_suffix if beta_suffix else ''
decoding_func = {
'gerrit': lambda content: base64.b64decode(content).decode('utf-8'),
'gitea': lambda content: base64.b64decode(json.loads(content.decode())
['content']).decode('utf-8'),
'github': lambda content: base64.b64decode(content).decode('utf-8'),
}
if 'gerrit' == git_type:
name = quote(name, safe='')
url = f'https://{git_url}/a/projects/{git_project}/branches/' \
f'{product_name[0].lower()}{distribution_major_version}.' \
f'{distribution_minor_version}' \
f'{beta_suffix}/files/{name}/content'
response = requests.get(url, auth=HTTPBasicAuth(
username=git_auth_username,
password=git_auth_token,
))
elif 'gitea' == git_type:
headers = {
'accept': 'application/json',
}
params = {
'access_token': git_auth_token,
'ref': (
f'{product_name[0].lower()}{distribution_major_version}.'
f'{distribution_minor_version}{beta_suffix}'
)
}
name = quote(name, safe='')
url = f'https://{git_url}/api/v1/repos/{git_project}/contents/{name}'
response = requests.get(url, params=params, headers=headers)
elif 'github' == git_type:
name = quote(name, safe='')
headers = {
'Authorization:': f'Bearer {git_auth_token}',
'Accept': 'application/vnd.github+json',
}
params = {
'ref': (
f'{product_name[0].lower()}{distribution_major_version}.'
f'{distribution_minor_version}{beta_suffix}'
)
}
url = f'https://api.github/repos/{git_project}/contents/{name}'
response = requests.get(url, params=params, headers=headers)
else:
raise NotImplemented(f'{git_type} is not supported yet')
try:
response.raise_for_status()
except requests.RequestException:
return
return decoding_func[git_type](response.content)
def render_variants_options(
env: jinja2.Environment,
variables: Dict,
) -> dict:
variants_options_template = env.get_template(os.path.join(
variables['arch'],
VARIANTS_OPTIONS_FILENAME,
))
variants_options = variants_options_template.render(**variables)
return json.loads(variants_options)
def render_pungi_build_conf(
env: jinja2.Environment,
variables: Dict,
) -> str:
pungi_build_conf_template = env.get_template(
PUNGI_BUILD_CONF_TEMPLATE_FILENAME,
)
return pungi_build_conf_template.render(**variables)
def prepare_build_conf(
product_name: str,
arch: str,
distribution_major_version: int,
distribution_minor_version: int,
env_path: str,
sigkeys_fingerprints: List[str],
git_url: str,
git_project: str,
git_auth_token: str,
git_auth_username: str,
git_type: str = 'gitea',
beta_suffix: str = '',
):
logging.info(
'Prepare build conf'
)
variables = {
'product_name': product_name,
'arch': arch,
'distribution_major_version': distribution_major_version,
'distribution_minor_version': distribution_minor_version,
'beta_suffix': beta_suffix,
'sigkeys_fingerprints': [
f'"{sigkey}"' for sigkey in sigkeys_fingerprints if sigkey
],
'git_auth_username': quote_plus(git_auth_username),
'git_auth_token': quote_plus(git_auth_token),
}
env = jinja2.Environment(
loader=jinja2.FunctionLoader(lambda name: load_remote_file_content(
name=name,
distribution_major_version=distribution_major_version,
distribution_minor_version=distribution_minor_version,
beta_suffix=beta_suffix,
git_url=git_url,
git_project=git_project,
git_auth_token=git_auth_token,
git_auth_username=git_auth_username,
product_name=product_name,
git_type=git_type,
)),
autoescape=jinja2.select_autoescape(),
trim_blocks=True,
lstrip_blocks=True,
)
variables['variants'] = render_variants_options(
env=env,
variables=variables,
)
extra_options_conf_lines = load_remote_file_content(
name=f'{arch}/{EXTRA_OPTIONS_CONF_FILENAME}',
distribution_minor_version=distribution_minor_version,
distribution_major_version=distribution_major_version,
beta_suffix=beta_suffix,
git_url=git_url,
git_project=git_project,
git_auth_token=git_auth_token,
git_auth_username=git_auth_username,
product_name=product_name,
git_type=git_type,
)
variables['extra_options'] = extra_options_conf_lines is not None
pungi_build_conf_lines = render_pungi_build_conf(
env=env,
variables=variables,
)
include_exclude_conf_lines = load_remote_file_content(
name=f'{arch}/{INCLUDE_EXCLUDE_CONF_FILENAME}',
distribution_minor_version=distribution_minor_version,
distribution_major_version=distribution_major_version,
beta_suffix=beta_suffix,
git_url=git_url,
git_project=git_project,
git_auth_token=git_auth_token,
git_auth_username=git_auth_username,
product_name=product_name,
git_type=git_type,
)
multilib_conf_lines = load_remote_file_content(
name=f'{arch}/{MULTILIB_CONF_FILENAME}',
distribution_minor_version=distribution_minor_version,
distribution_major_version=distribution_major_version,
beta_suffix=beta_suffix,
git_url=git_url,
git_project=git_project,
git_auth_token=git_auth_token,
git_auth_username=git_auth_username,
product_name=product_name,
git_type=git_type,
)
with open(os.path.join(
env_path,
PUNGI_BUILD_CONF_FILENAME,
), 'w') as fd:
fd.write(pungi_build_conf_lines)
with open(os.path.join(
env_path,
INCLUDE_EXCLUDE_CONF_FILENAME,
), 'w') as fd:
fd.write(include_exclude_conf_lines)
with open(os.path.join(
env_path,
MULTILIB_CONF_FILENAME,
), 'w') as fd:
fd.write(multilib_conf_lines)
if variables['extra_options']:
with open(os.path.join(
env_path,
EXTRA_OPTIONS_CONF_FILENAME,
), 'w') as fd:
fd.write(extra_options_conf_lines)
def add_koji_profile(
env_path: str,
profile_name: str,
):
logging.info(
'Add koji profile "%s" to "%s"',
profile_name,
KOJI_CONF_PATH,
)
koji_env_path = os.path.join(
env_path,
'koji'
)
with open(KOJI_CONF_PATH, 'r') as koji_conf_file:
koji_conf_obj = ConfigParser()
koji_conf_obj.read_file(koji_conf_file)
if profile_name not in koji_conf_obj.sections():
koji_conf_obj.add_section(profile_name)
koji_conf_obj.set(profile_name, 'topdir', koji_env_path)
with open(KOJI_CONF_PATH, 'w') as koji_conf_file:
koji_conf_obj.write(koji_conf_file)
def save_additional_env_files(
env_path: str,
add_env_files: List[str],
) -> None:
for add_env_file in filter(lambda i: True if i else False, add_env_files):
env_file_name, file_content_in_base64 = add_env_file.split(',')
env_file_content = base64.b64decode(
file_content_in_base64,
).decode('utf-8')
env_file_path = os.path.join(
env_path,
env_file_name,
)
with open(env_file_path, 'w') as env_file:
env_file.write(env_file_content)
def create_parser():
parser = argparse.ArgumentParser()
parser.add_argument(
'--env-path',
action='store',
help='A path to folder which will be used '
'for building new distribution',
required=False,
default=None,
)
subparsers = parser.add_subparsers(
dest='command',
)
parser_synchronize_using_dnf_reposync = subparsers.add_parser(
'dnf_reposync_synchronize',
help='Run synchronize using DNF reposync',
)
parser_synchronize_using_dnf_reposync.add_argument(
'--mirroring-dnf-repos',
nargs='*',
type=str,
help='A list of repos which will be mirrored',
required=False,
)
parser_synchronize_using_dnf_reposync.add_argument(
'--mirroring-target',
action='store',
help='A folder which will contain a local mirror',
required=False,
)
parser_synchronize_using_dnf_reposync.add_argument(
'--use-products-repos',
action='store_true',
default=False,
)
parser_synchronize_using_dnf_reposync.add_argument(
'--product-name',
action='store',
help='A name of building product',
required=True,
)
parser_synchronize_using_dnf_reposync.add_argument(
'--arch',
help='Architecture of a product',
action='store',
required=True,
)
parser_synchronize_using_dnf_reposync.add_argument(
'--distribution-major-version',
help='Major version of a product',
action='store',
required=True,
)
parser_add_env_files = subparsers.add_parser(
'add_env_files',
help='Save environment files which are passed as base64 strings'
)
parser_add_env_files.add_argument(
'--env-files',
nargs='*',
type=str,
help='A list of files which should be stored in env '
'directory. E.t. `add-comps.xml,<content_in_base64>`',
required=False,
default=[],
)
parser_koji_profile = subparsers.add_parser(
'add_koji_profile',
help=f'Add new koji profile to {KOJI_CONF_PATH}',
)
parser_koji_profile.add_argument(
'--koji-profile-name',
action='store',
help='A name of koji profile',
required=True,
)
parser_build_conf = subparsers.add_parser(
'prepare_build_conf',
help='Prepare a Pungi build conf',
)
parser_build_conf.add_argument(
'--product-name',
action='store',
help='A name of building product',
required=True,
)
parser_build_conf.add_argument(
'--arch',
help='Architecture of a product',
action='store',
required=True,
)
parser_build_conf.add_argument(
'--distribution-major-version',
help='Major version of a product',
action='store',
required=True,
)
parser_build_conf.add_argument(
'--distribution-minor-version',
help='Minor version of a product',
action='store',
required=True,
)
parser_build_conf.add_argument(
'--beta-suffix',
help='Suffix of a ISOs & Volume ID names. E.g. `-beta-1`',
action='store',
default='',
type=str,
required=False,
)
parser_build_conf.add_argument(
'--sigkeys-fingerprints',
nargs='*',
type=str,
help='A list of fingerprints of AlmaLinux sign keys. '
'They are used for checking that all packages are signed',
)
parser_build_conf.add_argument(
'--git-auth-token',
action='store',
type=str,
help='Auth token for access to a Git repository which '
'contains a build config and related stuff'
)
parser_build_conf.add_argument(
'--git-auth-username',
action='store',
type=str,
help='Auth username for access to a Git repository which '
'contains a build config and related stuff'
)
parser_build_conf.add_argument(
'--git-url',
action='store',
type=str,
help='Git URL for a Git repository which '
'contains a build config and related stuff'
)
parser_build_conf.add_argument(
'--git-project',
action='store',
type=str,
help='Name of a Git repository which '
'contains a build config and related stuff'
)
parser_build_conf.add_argument(
'--git-type',
action='store',
type=str,
default='gitea',
help='Type of a Git repository which '
'contains a build config and related stuff'
)
return parser
def check_is_root():
if os.geteuid():
logging.error('The script should be ran under root or any sudo user')
exit(1)
def cli_main():
# check_is_root()
parser = create_parser()
args = parser.parse_args()
if args.command == 'dnf_reposync_synchronize':
dnf_reposync_mirroring(
use_products_repos=args.use_products_repos,
product_name=args.product_name,
distribution_major_version=args.distribution_major_version,
arch=args.arch,
mirroring_target=args.mirroring_target,
)
else:
# another commands which require an env path
os.makedirs(args.env_path, exist_ok=True)
if args.command == 'add_env_files':
save_additional_env_files(
env_path=args.env_path,
add_env_files=args.env_files,
)
if args.command == 'add_koji_profile':
add_koji_profile(
env_path=args.env_path,
profile_name=args.koji_profile_name,
)
if args.command == 'prepare_build_conf':
prepare_build_conf(
beta_suffix=args.beta_suffix,
product_name=args.product_name,
arch=args.arch,
distribution_major_version=args.distribution_major_version,
distribution_minor_version=args.distribution_minor_version,
env_path=args.env_path,
sigkeys_fingerprints=args.sigkeys_fingerprints,
git_url=args.git_url,
git_project=args.git_project,
git_auth_token=args.git_auth_token,
git_auth_username=args.git_auth_username,
git_type=args.git_type,
)
if __name__ == '__main__':
cli_main()