From 08b75a7b93d38cb1ba58aba446a186284f74befc Mon Sep 17 00:00:00 2001 From: Trevor Batley Date: Tue, 9 May 2023 17:46:24 +1000 Subject: [PATCH] take out dangerous 'rm RF's --- git-cvs2git.sh | 149 +++++++++++++++++++++++++++++-------------------- 1 file changed, 88 insertions(+), 61 deletions(-) diff --git a/git-cvs2git.sh b/git-cvs2git.sh index 7153fc0..6217ad1 100755 --- a/git-cvs2git.sh +++ b/git-cvs2git.sh @@ -29,7 +29,6 @@ if [ ! -e $inifilename ] ; then echo "ERROR*******No ini file found $inifilename" exit 1 fi -echo "************found ini file: $inifilename" while read -r line || [[ -n "$line" ]]; do if [[ $line =~ ^\[.*\]$ ]] @@ -48,9 +47,14 @@ done < "$inifilename" DEBUG=false if [ ${smegit_DEBUG} == "true" ] ; then DEBUG=true ; fi +if [ $DEBUG ] ; then echo "************found ini file: $inifilename" ; fi # Make this null if you want lots of output. Still quite a bit anyway QUIET="-q" -if [ $DEBUG ] ; then QUIET=null ; fi +curlsilent="-s" +if [ $DEBUG ] ; then + QUIET= + curlsilent= +fi # Make this null to get curl to check ssl cert checkSSL="-k" @@ -86,7 +90,7 @@ mkdir -p $CVSFiles # Check that git-lfs is installed if command -v git-lfs > /dev/null; then - echo "************Git LFS is installed" + if [ $DEBUG ] ; then echo "************Git LFS is installed" ; fi else echo "ERROR********************** Git LFS is not installed ******************" exit 1 @@ -94,7 +98,7 @@ fi # Check that jq is installed if command -v jq -V > /dev/null; then - echo "************Jq is installed" + if [ $DEBUG ] ; then echo "************Jq is installed" ; fi else echo "ERROR********************** jq is not installed (try EPEL)**************" exit 1 @@ -103,14 +107,14 @@ fi # and check for the special scripts - changelog and BogusDateBot.sh - I think these are available in the # mockbuild rpm, but beware Bug:12340 - changelog needs an edit to work! if type -P change-log >/dev/null 2>&1; then - echo "************change-log is installed" + if [ $DEBUG ] ; then echo "************change-log is installed" ; fi else echo "ERROR********************change-log is not installed*********** (try installing smeserver-mockbuild)" fi -echo "************..and also note bug:12340". +if [ $DEBUG ] ; then echo "************..and also note bug:12340" ; fi if type -P BogusDateBot.sh >/dev/null 2>&1; then - echo "************BogusDateBot.sh is installed" + if [ $DEBUG ] ; then echo "************BogusDateBot.sh is installed"; fi else echo "ERROR********************BogusDateBot.sh is not installed*********** (try installing smeserver-mockbuild)" fi @@ -126,8 +130,16 @@ cd $CVSFiles # First delete any existing module directory tree # if there at all if [ -d "$1" ] ; then - echo "************Deleting local CVS files" - rm -Rf $1 + if [ $DEBUG ] ; then echo "************Deleting local CVS files" ; fi + cd $CVSFiles/$1 + # Delete all files, links and directories + find . -maxdepth 999 -xtype l -delete + find . -maxdepth 999 -type f -delete + # we seem to need this as well + find . -xtype l -exec rm -f {} + + find . -maxdepth 999 -type d -delete + cd $CVSFiles +# rm -Rf $1 fi @@ -195,8 +207,16 @@ cd $GITFiles # Delete the local first if [ -d "$1" ] ; then - echo "************Deleting local GIT files" - rm -Rf $1 + if [ $DEBUG ] ; then echo "************Deleting local GIT files" ; fi + cd $GITFiles/$1 + find . -maxdepth 999 -xtype l -delete + find . -maxdepth 999 -type f -delete + # we seem to need this as well + find . -xtype l -exec rm -f {} + + find . -maxdepth 999 -type d -delete + cd $GITFiles + rmdir $1 +# rm -Rf $1 fi @@ -204,7 +224,7 @@ fi REPO_NAME=$1 RESPONSE=$(curl "$checkSSL" -s -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$REPO_NAME") if [ "$RESPONSE" == "200" ]; then - echo "************Repository for $1 already exists!" + if [ $DEBUG ] ; then echo "************Repository for $1 already exists!" ; fi # If so, clone it (just want the README.md) # And Clone it cd $GITFiles @@ -212,7 +232,7 @@ if [ "$RESPONSE" == "200" ]; then #GIT_SSL_NO_VERIFY=true git clone "$GITEAHOST/$ORGGITEA/$1.git" NOREPO=0 else - echo "************Repository for $1 does not exist." + if [ $DEBUG ] ; then echo "************Repository for $1 does not exist." ; fi NOREPO=1 fi @@ -227,11 +247,18 @@ if [ -d "$1" ] ; then READMECONTENTS=`mktemp` mv -f README.md "$READMECONTENTS" fi - echo "************Saved README: $READMECONTENTS" + if [ $DEBUG ] ; then echo "************Saved README: $READMECONTENTS" ; fi cd $GITFiles # Delete all files, links and directories - echo "************Deleting local GIT files" - rm -Rf $1 + if [ $DEBUG ] ; then echo "************Deleting local GIT files" ; fi + find . -maxdepth 999 -xtype l -delete + find . -maxdepth 999 -type f -delete + # we seem to need this as well + find . -xtype l -exec rm -f {} + + find . -maxdepth 999 -type d -delete + cd $GITFiles + rmdir $1 +# rm -Rf $1 fi @@ -239,18 +266,18 @@ cd $GITFiles #git init $QUIET $1 # Now delete the remote repo and re-create it and clone it in to local -curl "$checkSSL" -X 'DELETE' \ +curl "$checkSSL" "$curlsilent" -X 'DELETE' \ "$GITEAHOST/api/v1/repos/$ORGGITEA/$1" \ -H 'accept: application/json' \ -H "Authorization: token $GITEAACCESSTOKEN" # and check if it is here. -RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$1") +RESPONSE=$(curl $curlsilent -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$1") if [ "$RESPONSE" == "200" ]; then - echo "************Repository for $1 exists!" + if [ $DEBUG ] ; then echo "************Repository for $1 exists!" ; fi GOTREPO=0 else - echo "************Repository for $1 does not exist" + if [ $DEBUG ] ; then echo "************Repository for $1 does not exist" ; fi GOTREPO=1 fi @@ -264,11 +291,11 @@ fi # # Use the API interface to create the remote repo # -echo "************Creating repo for $1 $PACKAGETYPE" +if [ $DEBUG ] ; then echo "************Creating repo for $1 $PACKAGETYPE" ; fi if [[ $GOTREPO -eq 1 ]] ; then #Create it if it does not exist - JSONREPO=$(curl "$checkSSL" -X 'POST' "$GITEAHOST/api/v1/orgs/$ORGGITEA/repos" \ + JSONREPO=$(curl "$checkSSL" "$curlsilent" -X 'POST' "$GITEAHOST/api/v1/orgs/$ORGGITEA/repos" \ -H 'accept: application/json' \ -H "Authorization: token $GITEAACCESSTOKEN" \ -H 'Content-Type: application/json' \ @@ -286,12 +313,12 @@ if [[ $GOTREPO -eq 1 ]] ; then "trust_model": "Default Trust Model" }') - echo $JSONREPO + if [ $DEBUG ] ; then echo $JSONREPO ; fi # and extract the http URL for it HTTP_URL=`echo $JSONREPO | jq -r '.html_url'` - echo $HTTP_URL + if [ $DEBUG ] ; then echo $HTTP_URL ; fi if [[ -z "${HTTP_URL}" ]]; then echo "ERROR*******Repo for $1 appears NOT to have been created " @@ -307,7 +334,7 @@ fi # And check it now exists RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$1") if [ "$RESPONSE" == "200" ]; then - echo "************Repository for $1 exists" + if [ $DEBUG ] ; then echo "************Repository for $1 exists" ; fi else echo "ERROR*******Repository for $1 has not been created ($JSONREQ)" exit 1 @@ -342,19 +369,19 @@ EXT=$(cat "$(basename $SPECFILE)" | grep -i ^Source | head -n 1 | sed -r 's/^[S| ARCHIVEFILE=$(cat "$(basename $SPECFILE)" | grep ^Source | head -n 1 | sed -r 's/^[S|s]ource[0..9]?:\s?//' | xargs basename) ARCHIVEFILE=$(rpm --queryformat "$ARCHIVEFILE\n" --specfile "$(basename $SPECFILE)" | head -n 1) -#echo $ARCHIVEFILE $EXT $SPECFILE +if [ $DEBUG ] ; then echo $ARCHIVEFILE $EXT $SPECFILE ; fi # Check that we got the filename (possible define in it not expanded?) if [ ! -f $ARCHIVEFILE ]; then # Try another tack - echo "************File $ARCHIVE does not exist - trying *.$EXT" + if [ $DEBUG ] ; then echo "************File $ARCHIVE does not exist - trying *.$EXT" ; fi ARCHIVEFILE=`ls *.$EXT | head -n 1` if [ ! -f $ARCHIVEFILE || -z $ARCHIVEFILE ]; then echo "ERROR*******Can't identify archive file $ARCHIVEFILE $EXT `pwd`" exit 1 fi fi -echo "************TreePATH:$TREEPATH and Archivefile:$ARCHIVEFILE `pwd`" +if [ $DEBUG ] ; then echo "************TreePATH:$TREEPATH and Archivefile:$ARCHIVEFILE `pwd`" ; fi # pull in all the files we want cd $GITFiles/$1 @@ -380,7 +407,7 @@ done # and see if the spec file is not according to specification of .spec # if not rename it if [[ $SPECFILE != $ORIGSPECFILE ]] ; then - echo "************Renaming specfile from $(basename $SPECFILE) $(basename $ORIGSPECFILE)" + if [ $DEBUG ] ; then echo "************Renaming specfile from $(basename $SPECFILE) $(basename $ORIGSPECFILE)" ; fi mv "$(basename $SPECFILE)" "$(basename $ORIGSPECFILE)" SPECFILE=$ORIGSPECFILE fi @@ -390,19 +417,19 @@ if [[ ! "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then # add in the patches (a 3rd party package) # it turns out that not all patches end in .patch, so extract the file names out of the spec file as per sources below PATCHS=$(grep -i -E '^patch[0-9]*:' $(basename $SPECFILE) | cut -d: -f2- | sed 's/ //g') - echo "$PATCHS" + if [ $DEBUG ] ; then echo "$PATCHS" ; fi for fpath in $PATCHS; do # Needs another step incase the extraction from the spec file contains spec variables fpathfile=$(basename $fpath) - echo "$fpathfile" + if [ $DEBUG ] ; then echo "$fpathfile" ; fi translatefpath=$(rpm --queryformat "$fpathfile\n" --specfile $(basename $SPECFILE) | head -n 1) - echo "$translatefpath" + if [ $DEBUG ] ; then echo "$translatefpath" ; fi if [[ -f $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ]] ; then - echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$translatefpath"; + if [ $DEBUG ] ; then echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$translatefpath" ; fi cp $CVSFiles/$1/$BASEORCONTRIB/$translatefpath . else - echo "************Not found $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ( $fpathfile )- so not copied" + if [ $DEBUG ] ; then echo "************Not found $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ( $fpathfile )- so not copied" ; fi fi done @@ -416,33 +443,33 @@ if [[ ! "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then # if the spec file includes "BuildRequires: perl(ExtUtils::MakeMaker)" and NOT "BuildRequires: perl(ExtUtils::Manifest)" then add the latter if grep -q "^BuildRequires:.*perl(ExtUtils::MakeMaker)" "$(basename $SPECFILE)" && ! grep -q "^BuildRequires:.*perl(ExtUtils::Manifest)" "$(basename $SPECFILE)"; then sed -i '/^BuildRequires:.*perl(ExtUtils::MakeMaker)/a BuildRequires: perl(ExtUtils::Manifest)' "$(basename $SPECFILE)" - echo "************Added 'BuildRequires: perl(ExtUtils::Manifest)' to the spec file." + if [ $DEBUG ] ; then echo "************Added 'BuildRequires: perl(ExtUtils::Manifest)' to the spec file." ; fi else - echo "************No action was taken. The spec file either does not include 'BuildRequires: perl(ExtUtils::MakeMaker)' or already includes 'BuildRequires: perl(ExtUtils::Manifest)'." + if [ $DEBUG ] ; then echo "************No action was taken. The spec file either does not include 'BuildRequires: perl(ExtUtils::MakeMaker)' or already includes 'BuildRequires: perl(ExtUtils::Manifest)'." ; fi fi # and similar for another ExtUtils package - ParseXS if grep -q "^BuildRequires:.*perl(ExtUtils::MakeMaker)" "$(basename $SPECFILE)" && ! grep -q "^BuildRequires:.*perl(ExtUtils::ParseXS)" "$(basename $SPECFILE)"; then sed -i '/^BuildRequires:.*perl(ExtUtils::MakeMaker)/a BuildRequires: perl(ExtUtils::ParseXS)' "$(basename $SPECFILE)" - echo "************Added 'BuildRequires: perl(ExtUtils::ParseXS)' to the spec file." + if [ $DEBUG ] ; then echo "************Added 'BuildRequires: perl(ExtUtils::ParseXS)' to the spec file." ; fi else - echo "************No action was taken. The spec file either does not include 'BuildRequires: perl(ExtUtils::MakeMaker)' or already includes 'BuildRequires: perl(ExtUtils::ParseXS)'." + if [ $DEBUG ] ; then echo "************No action was taken. The spec file either does not include 'BuildRequires: perl(ExtUtils::MakeMaker)' or already includes 'BuildRequires: perl(ExtUtils::ParseXS)'." ; fi fi # and similar for another devel package - needs manifest if grep -q "^BuildRequires:.*perl-devel" "$(basename $SPECFILE)" && ! grep -q "^BuildRequires:.*perl(ExtUtils::Manifest)" "$(basename $SPECFILE)"; then sed -i '/^BuildRequires:.*perl-devel/a BuildRequires: perl(ExtUtils::Manifest)' "$(basename $SPECFILE)" - echo "************Added 'BuildRequires: perl(ExtUtils::Manifest)' to the spec file." + if [ $DEBUG ] ; then echo "************Added 'BuildRequires: perl(ExtUtils::Manifest)' to the spec file." ; fi else - echo "************No action was taken. The spec file either does not include 'BuildRequires: perl-devel' or already includes 'BuildRequires: perl(ExtUtils::Manifest)'." + if [ $DEBUG ] ; then echo "************No action was taken. The spec file either does not include 'BuildRequires: perl-devel' or already includes 'BuildRequires: perl(ExtUtils::Manifest)'." ; fi fi # Fix the rm in the pear packages - .filemap is no longer part of the download if grep -q "^rm %{buildroot}/%{peardir}/.filemap" "$(basename $SPECFILE)"; then sed -i 's/^rm *\%/rm -rf %/' "$(basename $SPECFILE)" - echo "************Added 'rm -rf %{buildroot}/%{peardir}/.filemap' to the spec file." + if [ $DEBUG ] ; then echo "************Added 'rm -rf %{buildroot}/%{peardir}/.filemap' to the spec file." ; fi else - echo "************No action was taken. The spec file either does not include 'rm %{buildroot}/%{peardir}/.filemap' or already includes it" + if [ $DEBUG ] ; then echo "************No action was taken. The spec file either does not include 'rm %{buildroot}/%{peardir}/.filemap' or already includes it" ; fi fi @@ -450,20 +477,20 @@ fi ## and there may be extra files that maybe needed, these can be identified from the "Source" spec in the .spec files SOURCES=$(grep -i -E '^Source[0-9]*:' $(basename $SPECFILE) | cut -d: -f2- | sed 's/ //g') -echo "$SOURCES" +if [ $DEBUG ] ; then echo "$SOURCES" ; fi #SOURCE=$(rpm --queryformat '$SOURCE\n' --specfile $SPECFILE | head -n 1) for fpath in $SOURCES; do # Needs another step incase the extraction from the spec file contains spec variables fpathfile=$(basename $fpath) - echo "$fpathfile" + if [ $DEBUG ] ; then echo "$fpathfile" ; fi translatefpath=$(rpm --queryformat "$fpathfile\n" --specfile $(basename $SPECFILE) | head -n 1) - echo "$translatefpath" + if [ $DEBUG ] ; then echo "$translatefpath" ; fi if [[ -f $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ]] ; then - echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$translatefpath"; + if [ $DEBUG ] ; then echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$translatefpath" ; fi cp $CVSFiles/$1/$BASEORCONTRIB/$translatefpath . else - echo "************Not found $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ( $fpathfile )- so not copied" + if [ $DEBUG ] ; then echo "************Not found $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ( $fpathfile )- so not copied" ; fi fi done @@ -477,12 +504,12 @@ cd $GITFiles/$1 # See if the package is a 3rd party or one of ours - indicated by "smeserver" or "e-smith" in the name if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then # one of ours - echo "************Found SMEServer source for $NAME" + if [ $DEBUG ] ; then echo "************Found SMEServer source for $NAME" ; fi # Create the .gitignore file to ignore the archive echo "*.$EXT" >> .gitignore #and copy in the source tree mkdir -p "$CODEDIR" - echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/root `pwd`/$CODEDIR" + if [ $DEBUG ] ; then echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/root `pwd`/$CODEDIR" ; fi rsync -a --no-prune-empty-dirs "$CVSFiles/$1/$BASEORCONTRIB/$TREEPATH/root/" "`pwd`/$CODEDIR/" # copy createlinks to base if [ -e "$CVSFiles/$1/$BASEORCONTRIB/$TREEPATH/createlinks" ] ; then cp "$CVSFiles/$1/$BASEORCONTRIB/$TREEPATH/createlinks" "`pwd`/." ; fi @@ -494,16 +521,16 @@ if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then else # Third party package # copy over the archive file and make sure the extension is tracked by git-lfs - echo "************Found 3rd party package $NAME" + if [ $DEBUG ] ; then echo "************Found 3rd party package $NAME" ; fi git lfs install git lfs track "*.$EXT" git add .gitattributes #git config lfs.http://gitea.bjsystems.co.uk/SMEContribs/bugzilla.git/info/lfs.locksverify true if [[ -f $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE ]] ; then - echo "************Copying archivefile $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE" + if [ $DEBUG ] ; then echo "************Copying archivefile $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE" ; fi cp $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE . else - echo "Unable to find archivefile: $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE - nothing copied" + if [ $DEBUG ] ; then echo "Unable to find archivefile: $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE - nothing copied" ; fi fi cd $GITFiles/$1 fi @@ -577,7 +604,7 @@ chmod +x $GITFiles/common/make-archive.sh # Edit the Readme.md - try to add a link to the wiki for this package - only if it is an SME package cd $GITFiles/$1 -echo "************READMECONTENTS:$READMECONTENTS" +if [ $DEBUG ] ; then echo "************READMECONTENTS:$READMECONTENTS" ; fi PACKAGENAME=${NAME} if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then # See if we have a previous contents of the README.md @@ -615,7 +642,7 @@ if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then # nothing previous # Copy across the Wiki links to the README if [ ! "$URLS" = "" ]; then - echo "************The wiki page(s) for $PACKAGENAME exist." + if [ $DEBUG ] ; then echo "************The wiki page(s) for $PACKAGENAME exist." ; fi echo -en "\n\n## Wiki\n$URLS" >> README.md fi @@ -641,7 +668,7 @@ if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then model="text-davinci-003" temperature=0.8 # Send the request to the API - response=$(curl -X POST https://api.openai.com/v1/engines/$model/completions \ + response=$(curl "$curlsilent" -X POST https://api.openai.com/v1/engines/$model/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $OPENAI_API_KEY" \ -d '{"prompt": "'"$prompt"'", "temperature": 0.8, "max_tokens": 256}') @@ -655,7 +682,7 @@ if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then echo -e "
$PARAGRAPH" >> README.md else # overwrite with previous contents - echo "************Picked up previous README contents" + if [ $DEBUG ] ; then echo "************Picked up previous README contents" ; fi mv -f $READMECONTENTS ./README.md fi else @@ -665,21 +692,21 @@ else model="text-davinci-003" temperature=0.8 # Send the request to the API - response=$(curl -X POST https://api.openai.com/v1/engines/$model/completions \ + response=$(curl "$curlsilent" -X POST https://api.openai.com/v1/engines/$model/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $OPENAI_API_KEY" \ -d '{"prompt": "'"$prompt"'", "temperature": 0.8, "max_tokens": 256}') # Extract the text from the response PARAGRAPH=`echo $response | jq -r '.choices[].text'` # Add it to what was there already - echo "************Creating description for $1" + if [ $DEBUG ] ; then echo "************Creating description for $1" ; fi echo -e "\n\n## Description\n" >> README.md echo -e "
*This description has been generated by an LLM AI system and cannot be relied on to be fully correct.*" >> README.md echo -e "*Once it has been checked, then this comment will be deleted*" >> README.md echo -e "
$PARAGRAPH" >> README.md else # overwrite with previous contents - echo "************Picked up previous README contents" + if [ $DEBUG ] ; then echo "************Picked up previous README contents" ; fi mv -f $READMECONTENTS ./README.md fi fi @@ -690,7 +717,7 @@ if [[ -z $WIKILINK ]] ; then fi # and then update the repo parameters -RESPONSE=$(curl -X 'PATCH' "$GITEAHOST/api/v1/repos/$ORGGITEA/$REPO_NAME" \ +RESPONSE=$(curl "$curlsilent" -X 'PATCH' "$GITEAHOST/api/v1/repos/$ORGGITEA/$REPO_NAME" \ -H 'accept: application/json' \ -H "Authorization: token $GITEAACCESSTOKEN" \ -H 'Content-Type: application/json' \ @@ -706,7 +733,7 @@ RESPONSE=$(curl -X 'PATCH' "$GITEAHOST/api/v1/repos/$ORGGITEA/$REPO_NAME" \ } ') # should check response here. -echo $RESPONSE +if [ $DEBUG ] ; then echo $RESPONSE ; fi #exit 0 cd $GITFiles/$1