fix lfs, Makefile, README.md and BuildRequires

This commit is contained in:
Trevor Batley 2024-03-27 15:05:25 +11:00
parent c5800c3e14
commit fb6a44f132

View File

@ -12,7 +12,7 @@
#
if [[ -z $1 ]] ; then
echo "************git-getperl.sh <modulename> <organization> [<local>]"
echo "************git-getperl.sh <modulename> <organization> [<local> | <debug> | <noisy>]"
exit 0
fi
@ -53,7 +53,9 @@ GITEAACCESSTOKEN=${remote_GITEAACCESSTOKEN}
GITEAHOST=${remote_GITEAHOST}
NOISY=
EXTRAPARAMS=
for param in $3 $4 $5 $6; do
MODULENAME=$1
ORGGITEA="smeserver"
for param in $2 $3 $4 $5 $6; do
if [ $param ] ; then
case $param in
local )
@ -66,7 +68,7 @@ for param in $3 $4 $5 $6; do
noisy )
NOISY=true ;;
* )
EXTRAPARAMS=$EXTRAPARAMS" "$param ;;
ORGGITEA=$param ;;
esac
else
break
@ -99,19 +101,21 @@ if [ $DEBUG ] ; then echo "WORKDIR=$WORKDIR" ; fi
# create repo name from perl- module with :: replaced by -
IFS=::
words=()
for i in $1; do words+=($i) ; done
for i in MODULENAME; do words+=($i) ; done
RepoName="perl"
for word in ${words[@]}; do RepoName="$RepoName-$word" ; done
ShortName=
for word in ${words[@]}; do
RepoName="$RepoName-$word"
if [ $ShortName ] ; then
ShortName="$ShortName-$word"
else
Shortname="$word"
fi
done
if [ $DEBUG ] ; then echo "RepoName=$RepoName" ; fi
IFS="${IFS:0:3}"
RemoteRepoURL="$GITEAHOST/$ORGGITEA/$RepoName"
#
# Unless you have changed this, this (below) is my (brian) OpenAI token, but happy to use it here, it costs peanuts!
# unless you delete the repo in between duplicate runs, then the README.md is preserved, so the ChatGPT is only called once.
# but it means you can edit the results on gitea without fear of loosing your edits if the repo is re-created using this script
#
export OPENAI_API_KEY=${smegit_OPENAI_API_KEY}
COMMONREPO=${smegit_COMMONREPO}
# Prepare variable for file in repo to indicate core or contrib.
@ -130,6 +134,14 @@ fi
# Work out if it is an SMEServer package or a 3rd party one
PACKAGETYPE="3rd Party (Maintained by Koozali)"
# check that cpanspec is installed
if command -v cpanspec >/dev/null; then
if [ $DEBUG ] ; then echo "************cpanspec is installed"; fi
else
echo "ERROR********************cpanspec is not installed*********** (try installing cpanspec)"
exit 1
fi
# Check that git-lfs is installed
if command -v git-lfs > /dev/null; then
if [ $DEBUG ] ; then echo "************Git LFS is installed" ; fi
@ -138,15 +150,6 @@ else
exit 1
fi
# Check that jq is installed
if command -v jq -V > /dev/null; then
if [ $DEBUG ] ; then echo "************Jq is installed" ; fi
else
echo "ERROR********************** jq is not installed (try EPEL)**************"
exit 1
fi
# Check that lua is installed (needed for change-log)
if command -v lua -V > /dev/null; then
if [ $DEBUG ] ; then echo "************lua is installed" ; fi
@ -155,7 +158,6 @@ else
exit 1
fi
# and check for the special scripts - changelog and BogusDateBot.sh - I think these are available in the
# mockbuild rpm, but beware Bug:12340 - changelog needs an edit to work!
if type -P change-log >/dev/null 2>&1; then
@ -166,13 +168,6 @@ else
fi
if [ $DEBUG ] ; then echo "************..and also note bug:12340" ; fi
if type -P BogusDateBot.sh >/dev/null 2>&1; then
if [ $DEBUG ] ; then echo "************BogusDateBot.sh is installed"; fi
else
echo "ERROR********************BogusDateBot.sh is not installed*********** (try installing smeserver-mockbuild)"
exit 1
fi
# Create work directories
cd $WORKDIR
perlFiles="$WORKDIR/$ORGGITEA/perl"
@ -196,7 +191,7 @@ fi
# Then get it fully from CVS
mkdir -p $perlFiles/$RepoName
cd $perlFiles/$RepoName
cpanspec "$1"
cpanspec "MODULENAME"
# Check that the spec file is there
# First bet is the packagename.spec
@ -279,7 +274,7 @@ else
fi
cd $GITFiles
#git init $QUIET $1
#git init $QUIET MODULENAME
#
# Re-create the repo
@ -305,16 +300,6 @@ RESPONSE=$(curl "$checkSSL" "$SILENT4CURL" -X 'POST' \
}')
if [ $DEBUG ] ; then echo $RESPONSE ; fi
# and extract the http URL for it
HTTP_URL=`echo $RESPONSE | jq -r '.html_url'`
if [ $DEBUG ] ; then echo $HTTP_URL ; fi
#if [[ -z "${HTTP_URL}" ]]; then
# echo "ERROR*******Repo for $RepoName appears NOT to have been created "
# echo "$RESPONSE"
# exit 1
#fi
# And check it now exists
RESPONSE=$(curl "$SILENT4CURL" -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$RepoName")
if [[ "$RESPONSE" == "200" ]]; then
@ -324,9 +309,31 @@ else
exit 1
fi
# Tell repo to use koozali.org bugzilla and wiki - this does not work on CREATE currently - 19Apr2023
WIKILINK="https://wiki.koozali.org/$RepoName"
# and then update the repo parameters
RESPONSE=$(curl "$SILENT4CURL" -X 'PATCH' "$GITEAHOST/api/v1/repos/$ORGGITEA/$RepoName" \
-H 'accept: application/json' \
-H "Authorization: token $GITEAACCESSTOKEN" \
-H 'Content-Type: application/json' \
-d '{
"external_tracker": {
"external_tracker_format": "https://bugs.koozali.org/show_bug.cgi?id={index}",
"external_tracker_style": "numeric",
"external_tracker_url": "'"https://bugs.koozali.org/buglist.cgi?component=$RepoName&product=$PRODUCTBUGZILLA"'"
},
"external_wiki": {
"external_wiki_url": "'"$WIKILINK"'"
}
}
')
# should check response here.
if [ $DEBUG ] ; then echo $RESPONSE ; fi
#exit 0
# Pull in the auto created or updated files
cd $GITFiles
#GIT_SSL_NO_VERIFY=true git clone "$GITEAHOST/$ORGGITEA/$RepoName.git"
git clone "$GITEAHOST/$ORGGITEA/$RepoName.git"
if [ ! -d $GITFiles/$RepoName ] ; then
@ -345,24 +352,24 @@ cat > Makefile <<- _EOT
# Makefile for source rpm: $RepoName
# $Id: Makefile,v 1.1 2016/02/04 12:24:52 vip-ire Exp $
NAME := $RepoName
SPECFILE = $(firstword $(wildcard *.spec))
SPECFILE = \$(firstword \$(wildcard *.spec))
define find-makefile-common
for d in common ../common ../../common ; do if [ -f $$d/Makefile.common ] ; then if [ -f $$d/CVS/Root -a -w $$/Makefile.common ] ; then cd $$d ; cvs -Q update ; fi ; echo "$$d/Makefile.common" ; break ; fi ; done
for d in common ../common ../../common ; do if [ -f \$\$d/Makefile.common ] ; then if [ -f \$\$d/CVS/Root -a -w \$\$/Makefile.common ] ; then cd \$\$d ; cvs -Q update ; fi ; echo "\$\$d/Makefile.common" ; break ; fi ; done
endef
MAKEFILE_COMMON := $(shell $(find-makefile-common))
MAKEFILE_COMMON := \$(shell \$(find-makefile-common))
ifeq ($(MAKEFILE_COMMON),)
ifeq (\$(MAKEFILE_COMMON),)
# attept a checkout
define checkout-makefile-common
test -f CVS/Root && { cvs -Q -d $$(cat CVS/Root) checkout common && echo "common/Makefile.common" ; } || { echo "ERROR: I can't figure out how to checkout the 'common' module." ; exit -1 ; } >&2
test -f CVS/Root && { cvs -Q -d \$\$(cat CVS/Root) checkout common && echo "common/Makefile.common" ; } || { echo "ERROR: I can't figure out how to checkout the 'common' module." ; exit -1 ; } >&2
endef
MAKEFILE_COMMON := $(shell $(checkout-makefile-common))
MAKEFILE_COMMON := \$(shell \$(checkout-makefile-common))
endif
include $(MAKEFILE_COMMON)
include \$(MAKEFILE_COMMON)
_EOT
# Add file indicating which release and contribs or base - this is matched by the makefile with the contents of the branch file in the common directory
@ -404,45 +411,24 @@ echo "*.log" >> .gitignore
echo "*spec-20*" >>.gitignore
cd $GITFiles/$RepoName
# See if the package is a 3rd party or one of ours - indicated by "smeserver" or "e-smith" in the name
if [[ "$RepoName" =~ ^(smeserver-|e-smith-).* ]]; then
# one of ours
if [ $DEBUG ] ; then echo "************Found SMEServer source for $NAME" ; fi
# Create the .gitignore file to ignore the archive
echo "*.$EXT" >> .gitignore
#and copy in the source tree
mkdir -p "$CODEDIR"
if [ $DEBUG ] ; then echo "************Copying $CVSFiles/$RepoName/$BASEORCONTRIB/root `pwd`/$CODEDIR" ; fi
rsync "$QUIET" -a --no-prune-empty-dirs "$CVSFiles/$RepoName/$BASEORCONTRIB/$TREEPATH/root/" "`pwd`/$CODEDIR/"
# copy createlinks to base
if [ -e "$CVSFiles/$RepoName/$BASEORCONTRIB/$TREEPATH/createlinks" ] ; then cp "$CVSFiles/$RepoName/$BASEORCONTRIB/$TREEPATH/createlinks" "`pwd`/." ; fi
# copy anything else in the base directory of $TREEPATH into the additional directory
if [ ! -d "`pwd`/additional" ] ; then mkdir -p "`pwd`/additional" ; fi
rsync "$QUIET" -av --exclude "createlinks" --exclude "root/" "$CVSFiles/$RepoName/$BASEORCONTRIB/$TREEPATH/" "`pwd`/additional/"
# and plug a dummy into any empty directories
find "`pwd`/root"/* -type d -empty -exec touch {}/.gitignore \;
else
# Third party package
# copy over the archive file and make sure the extension is tracked by git-lfs
if [ $DEBUG ] ; then echo "************Found 3rd party package $NAME" ; fi
git lfs install
git lfs track "*.$EXT"
git add .gitattributes
#git config lfs.http://gitea.bjsystems.co.uk/SMEContribs/bugzilla.git/info/lfs.locksverify true
if [[ -f $CVSFiles/$RepoName/$BASEORCONTRIB/$ARCHIVEFILE ]] ; then
# All perl packages are Third party packages
# copy over the archive file and make sure the extension is tracked by git-lfs
if [ $DEBUG ] ; then echo "************Found 3rd party package $NAME" ; fi
git lfs install
git lfs track "*.$EXT"
git add .gitattributes
#git config lfs.http://gitea.bjsystems.co.uk/SMEContribs/bugzilla.git/info/lfs.locksverify true
if [[ -f $CVSFiles/$RepoName/$BASEORCONTRIB/$ARCHIVEFILE ]] ; then
if [ $DEBUG ] ; then echo "************Copying archivefile $CVSFiles/$RepoName/$BASEORCONTRIB/$ARCHIVEFILE" ; fi
cp $CVSFiles/$RepoName/$BASEORCONTRIB/$ARCHIVEFILE .
else
else
if [ $DEBUG ] ; then echo "Unable to find archivefile: $CVSFiles/$RepoName/$BASEORCONTRIB/$ARCHIVEFILE - nothing copied" ; fi
fi
cd $GITFiles/$RepoName
fi
cd $GITFiles/$RepoName
# Take out any bogus dates:
#BogusDateBot.sh $(basename $SPECFILE)
# Add the BuildRequires now needed by el8(Rocky 8)
sed -i -e '/BuildRequires/{a\BuildRequires: perl(inc::Module::Install)' -e ':a;n;ba}' "$SPECFILE"
sed -i -e '/BuildRequires/{a\BuildRequires: perl(ExtUtils::Manifest)' -e ':a;n;ba}' "$SPECFILE"
# Re-factor the Common directory so that it is outside the repo being created.
# one Common directory for each Organisation
@ -464,137 +450,23 @@ fi
# Edit the Readme.md - try to add a link to the wiki for this package - only if it is an SME package
cd $GITFiles/$RepoName
if [ $DEBUG ] ; then echo "************READMECONTENTS:$READMECONTENTS" ; fi
PACKAGENAME=${NAME}
if [[ "$RepoName" =~ ^(smeserver-|e-smith-).* ]]; then
# See if we have a previous contents of the README.md
if [[ "$RepoName" =~ ^(smeserver-).* ]]; then
PACKAGENAME=${NAME#smeserver-}
fi
DESCRIPTION=`rpm --queryformat '%{description}\n' --specfile "$SPECFILE"
cat > README.md <<- _EOT
# $RepoName
# first find the relevant wiki entry
SMEServer Koozali local git repo for $RepoName
MEDIAWIKI_SITE="https://wiki.koozali.org/api.php"
SEARCH_TERM="$PACKAGENAME"
## Documentation
https://metacpan.org/pod/$MODULENAME
# Call the API to perform the search and store the JSON response in a variable
RESPONSE=$(curl -s "$MEDIAWIKI_SITE?action=query&format=json&list=search&srsearch=${SEARCH_TERM}&srprop=size%7Cwordcount%7Ctimestamp%7Csnippet&srlimit=10")
## Bugs
CPAN bug report are [here](https://rt.cpan.org/Public/Dist/Display.html?Name=${$ShortName})
# Use jq to extract the titles and pageids of the pages that match the search term (case-insensitive)
RESULTS=$(echo "$RESPONSE" | jq -r '.query.search[] | select(.title | ascii_downcase | contains("'"${SEARCH_TERM}"'" | ascii_downcase)) | .title, .pageid')
Show list of Koozali outstanding bugs: [here](https://bugs.koozali.org/buglist.cgi?component=$RepoName&product=SME%20Server%2011.X&query_format=advanced&limit=0&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&bug_status=CONFIRMED)
# Loop through the results and construct the URL for each page
URLS=$(while read -r TITLE; do \
read -r PAGEID; \
URL="https://wiki.koozali.org/${TITLE}"; \
URL=$(echo $URL | sed 's/ /_/g'); \
echo "<br />${URL}"; \
done <<< "${RESULTS}" )
# and get the first non french (sorry JP!)
WIKILINK=$(while read -r TITLE; do \
read -r PAGEID; \
URL="https://wiki.koozali.org/${TITLE}"; \
URL=$(echo $URL | sed 's/ /_/g'); \
if [[ ! "$URL" =~ 'fr' ]] ; then echo "${URL}"; break; fi \
done <<< "${RESULTS}" )
if [[ "$READMECONTENTS" == "" ]]; then
# nothing previous
# Copy across the Wiki links to the README
if [ ! "$URLS" = "" ]; then
if [ $DEBUG ] ; then echo "************The wiki page(s) for $PACKAGENAME exist." ; fi
echo -en "\n\n## Wiki\n$URLS" >> README.md
fi
# And add bugzilla link
echo -en "\n\n## Bugzilla" >> README.md
echo -en "\nShow list of outstanding bugs: [here](https://bugs.koozali.org/buglist.cgi?component=$RepoName&product=$PRODUCTBUGZILLA&query_format=advanced&limit=0&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&bug_status=CONFIRMED)" >> README.md
# Edit the header to add the logo
#KOOZALILOGO='<img src="http:\/\/vps.bjsystems.co.uk\/KoozaliLogo4GiteaRepo.png" style="vertical-align:bottom">'
KOOZALILOGO='<img src="https:\/\/www.koozali.org\/images\/koozali\/Logo\/Png\/Koozali_logo_2016.png" width="25%" vertical="auto" style="vertical-align:bottom">'
if [[ "$RepoName" =~ ^(smeserver-).* ]]; then
sed -i "s/^# smeserver/# $KOOZALILOGO smeserver/" README.md
else
sed -i "s/^# e-smith/# $KOOZALILOGO e-smith/" README.md
fi
# Good to extract some of the wiki here, and load it into the readme,
# OR I could ask GPTChat to write me paragraph on the contrib 8-))
# Set the parameters for the request
if [[ "$RepoName" =~ ^(smeserver-).* && "$ORGGITEA" == "smecontribs" ]]; then
prompt="Write me a paragraph on the $PACKAGENAME software contrib for smeserver"
else
prompt="Write me a paragraph about the software $PACKAGENAME"
fi
model="text-davinci-003"
temperature=0.8
# Send the request to the API
response=$(curl "$SILENT4CURL" -X POST https://api.openai.com/v1/engines/$model/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $OPENAI_API_KEY" \
-d '{"prompt": "'"$prompt"'", "temperature": 0.8, "max_tokens": 256}')
# Extract the text from the response
PARAGRAPH=`echo $response | jq -r '.choices[].text'`
# Add it to what was there already
if [ $DEBUG ] ; then echo "************Creating description for $RepoName" ; fi
echo -e "\n\n## Description\n" >> README.md
echo -e "<br />*This description has been generated by an LLM AI system and cannot be relied on to be fully correct.*" >> README.md
echo -e "*Once it has been checked, then this comment will be deleted*" >> README.md
echo -e "<br />$PARAGRAPH" >> README.md
else
# overwrite with previous contents
if [ $DEBUG ] ; then echo "************Picked up previous README contents" ; fi
mv -f $READMECONTENTS ./README.md
fi
else
if [ "$READMECONTENTS" == "" ]; then
# Add in at least a description
prompt="Tell me what the software package $PACKAGENAME does?"
model="text-davinci-003"
temperature=0.8
# Send the request to the API
response=$(curl "$SILENT4CURL" -X POST https://api.openai.com/v1/engines/$model/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $OPENAI_API_KEY" \
-d '{"prompt": "'"$prompt"'", "temperature": 0.8, "max_tokens": 256}')
# Extract the text from the response
PARAGRAPH=`echo $response | jq -r '.choices[].text'`
# Add it to what was there already
if [ $DEBUG ] ; then echo "************Creating description for $RepoName" ; fi
echo -e "\n\n## Description\n" >> README.md
echo -e "<br />*This description has been generated by an LLM AI system and cannot be relied on to be fully correct.*" >> README.md
echo -e "*Once it has been checked, then this comment will be deleted*" >> README.md
echo -e "<br />$PARAGRAPH" >> README.md
else
# overwrite with previous contents
if [ $DEBUG ] ; then echo "************Picked up previous README contents" ; fi
mv -f $READMECONTENTS ./README.md
fi
fi
# Tell repo to use koozali.org bugzilla and wiki - this does not work on CREATE currently - 19Apr2023
if [[ -z $WIKILINK ]] ; then
WIKILINK="https://wiki.koozali.org"
fi
# and then update the repo parameters
RESPONSE=$(curl "$SILENT4CURL" -X 'PATCH' "$GITEAHOST/api/v1/repos/$ORGGITEA/$REPO_NAME" \
-H 'accept: application/json' \
-H "Authorization: token $GITEAACCESSTOKEN" \
-H 'Content-Type: application/json' \
-d '{
"external_tracker": {
"external_tracker_format": "https://bugs.koozali.org/show_bug.cgi?id={index}",
"external_tracker_style": "numeric",
"external_tracker_url": "'"https://bugs.koozali.org/buglist.cgi?component=$REPO_NAME&product=$PRODUCTBUGZILLA"'"
},
"external_wiki": {
"external_wiki_url": "'"$WIKILINK"'"
}
}
')
# should check response here.
if [ $DEBUG ] ; then echo $RESPONSE ; fi
#exit 0
## Description
$DESCRIPTION
_EOT
cd $GITFiles/$RepoName
# stage and commit all the files created locally
@ -602,13 +474,12 @@ git add -A
echo "$RepoName git status" >>gitstatus.log
git status >>gitstatus.log
if [ $DEBUG ] ; then git status ; fi
COMMENT="initial commit of file from CVS for $RepoName on `date`"
COMMENT="initial create of smeserevr perl $MODULENAME on `date`"
git commit -m "$COMMENT" $QUIET
# push the converted git repository to the remote
git remote remove origin
git remote add origin "$GITEAHOST/$ORGGITEA/$RepoName.git"
#GIT_SSL_NO_VERIFY=true git push --force -u $QUIET origin master
git push --force -u $QUIET origin master
# Now create the version and release tag from the specfile
@ -625,9 +496,9 @@ if [ $DEBUG ] ; then
echo "TAG=$TAG"
fi
git pull
git tag -a $TAG -m "Setting tag to current Version-Release in spec file: $TAG"
git tag -a $TAG -m "align tag with version and release from spec file: $TAG"
git push origin --tag $TAG $QUIET
echo "Created $RepoName Repo: $HTTP_URL - Comment:$COMMENT tag:$TAG"
echo "$COMMENT tag:$TAG"
exit 0