From e6d8326770aecd4534879245fb2204938dd34614 Mon Sep 17 00:00:00 2001 From: Trevor Batley Date: Tue, 9 May 2023 13:34:39 +1000 Subject: [PATCH] Initial load --- README.md | 1 + git-cvs2git.sh | 735 ++++++++++++++++++++++++++++++++++++++ git-get-repo-and-build.sh | 144 ++++++++ git-mirror-org.sh | 65 ++++ git-mirror-repo.sh | 100 ++++++ 5 files changed, 1045 insertions(+) create mode 100644 README.md create mode 100755 git-cvs2git.sh create mode 100755 git-get-repo-and-build.sh create mode 100755 git-mirror-org.sh create mode 100755 git-mirror-repo.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..9b459c4 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +Utilities to use with smeserver on Koozali.org GITEA instamce \ No newline at end of file diff --git a/git-cvs2git.sh b/git-cvs2git.sh new file mode 100755 index 0000000..9d8ee39 --- /dev/null +++ b/git-cvs2git.sh @@ -0,0 +1,735 @@ +#!/bin/bash +# +# $1 = Module name e.g. smeserver-ddclient +# $2 = Organisation (smeserver or smecontrib)e +# $3 = "local" will use parameters set for local repository else it will use remote +# This works whether the repo and local files exist of not (it deletes them if necessary) +# However if the remote repo exists, it preserves the README.md file so that edits are not lost +# Also note: I have had difficulty deleting all the link files in the source tree! +# + +if [[ -z $1 ]] ; then + echo "************git_cvs2git.sh []" + exit 0 +fi +clear +echo "**************************************************************************************************" +echo "* *" +echo "* SMEServer - git_cvs2git $1 $2 $3 `date` *" +echo "* *" +echo "**************************************************************************************************" + +# +# Pull in parameters from a config file ~/.smegit/config +# +#inifilename=$(find ~/GITFiles -name cvs2git.ini 2>/dev/null | head -n 1) +inifilename=$(echo ~)"/.smegit/config" +if [ ! -e $inifilename ] ; then + # Not here + echo "ERROR*******No ini file found $inifilename" + exit 1 +fi +echo "************found ini file: $inifilename" + +while read -r line || [[ -n "$line" ]]; do + if [[ $line =~ ^\[.*\]$ ]] + then + section=${line#*[} + section=${section%]*} + else + if [[ $line =~ ^[^#]*= ]] + then + key=${line%=*} + value=${line#*=} + declare "${section}_${key}=$value" + fi + fi +done < "$inifilename" +DEBUG=false +if [ ${smegit_DEBUG} == "true" ] ; then DEBUG=true ; fi + +# Make this null if you want lots of output. Still quite a bit anyway +QUIET="-q" +if [ $DEBUG ] ; then QUIET=null ; fi +# Make this null to get curl to check ssl cert +checkSSL="-k" + +LOCALUser=${local_USER} +WORKDIR=${smegit_WORKDIR} +GITEAUser=${remote_USER} +GITEAACCESSTOKEN=${remote_GITEAACCESSTOKEN} +GITEAHOST=${remote_GITEAHOST} +if [ $3 ] ; then +if [ $3 == "local" ] ; then + GITEAUser=${local_USER} + GITEAACCESSTOKEN=${local_GITEAACCESSTOKEN} + GITEAHOST=${local_GITEAHOST} +fi +RemoteRepoURL="$GITEAHOST/$2/$1" + +# +# Unless you have changed this, this (below) is my (brian) OpenAI token, but happy to use it here, it costs peanuts! +# unless you delete the repo in between duplicate runs, then the README.md is preserved, so the ChatGPT is only called once. +# but it means you can edit the results on gitea without fear of loosing your edits if the repo is re-created using this script +# +export OPENAI_API_KEY=${smegit_OPENAI_API_KEY} +COMMONREPO=${smegit_COMMONREPO} + +# Name is re-initialised later on from the spec file +NAME=$1 + +cd $WORKDIR +CVSFiles="$WORKDIR/$2/rpms" +#echo $CVSFiles +mkdir -p $CVSFiles + +# Check that git-lfs is installed +if command -v git-lfs > /dev/null; then + echo "************Git LFS is installed" +else + echo "ERROR********************** Git LFS is not installed ******************" + exit 1 +fi + +# Check that jq is installed +if command -v jq -V > /dev/null; then + echo "************Jq is installed" +else + echo "ERROR********************** jq is not installed (try EPEL)**************" + exit 1 +fi + +# and check for the special scripts - changelog and BogusDateBot.sh - I think these are available in the +# mockbuild rpm, but beware Bug:12340 - changelog needs an edit to work! +if type -P change-log >/dev/null 2>&1; then + echo "************change-log is installed" +else + echo "ERROR********************change-log is not installed*********** (try installing smeserver-mockbuild)" +fi +echo "************..and also note bug:12340". + +if type -P BogusDateBot.sh >/dev/null 2>&1; then + echo "************BogusDateBot.sh is installed" +else + echo "ERROR********************BogusDateBot.sh is not installed*********** (try installing smeserver-mockbuild)" +fi + +# Make sure credentials taken from store +# The first time you pull or push it will ask for credentials and then save them in a file. +git config --global credential.helper store + +# +# Clone the CVS repository +# +cd $CVSFiles +# First delete any existing module directory tree +# if there at all +if [ -d "$1" ] ; then + echo "************Deleting local CVS files" + rm -Rf $1 +fi + + +# Then get it fully from CVS +cd $CVSFiles +cvs -d:ext:shell.koozali.org:/cvs/$2 $QUIET checkout $1 + +# Prepare variable for file in repo to indicate core or contrib. +# and init gitea organisation +# Adjust where we put the repo locally accordingly. +ORGGITEA=$2 +if [ "$2" == "smecontribs" ]; then + BASEORCONTRIB="contribs10" + PRODUCTBUGZILLA="SME%20Contribs" +else + BASEORCONTRIB="sme10" + PRODUCTBUGZILLA="SME%20Server%2010.X" +fi + + +# Check that the spec file is there +# First bet is the packagename.spec +SPECFILE=$1/$BASEORCONTRIB/$1.spec +ORIGSPECFILE=$SPECFILE +GOTONE=0 +if [ ! -f $SPECFILE ] ; then + # See if it is a perl package which has the perl or perl-pear taken off the front + for front in "perl-" "perl-pear-" ; do + reg="^($front).*" + echo "$front $1 $reg" + if [[ "$1" =~ $reg ]]; then + NEWNAME="${1#$front}" + #echo $NEWNAME + SPECFILE="$1/$BASEORCONTRIB/$NEWNAME.spec" + #echo $SPECFILE + if [ -f $SPECFILE ] ; then + GOTONE=1 + break + fi + fi + done + # Otherwise, just pull out the first spec file in the directory + cd $1/$BASEORCONTRIB + SPECFILE=$(ls *.spec | head -n 1) + SPECFILE=$1/$BASEORCONTRIB/$SPECFILE + cd $CVSFiles + echo '2nd try: '$SPECFILE + if [ -f $SPECFILE ] ; then + GOTONE=1 + ORIGSPECFILE=$SPECFILE + fi +else + GOTONE=1 +fi + +if [[ $GOTONE -eq 0 ]] ; then + echo "Cannot find .spec file: `pwd`/$ORIGSPECFILE" + exit 1 +fi + +# Create the local Git repository +GITFiles=$WORKDIR/GITFiles/$ORGGITEA +mkdir -p $GITFiles +cd $GITFiles + +# Delete the local first +if [ -d "$1" ] ; then + echo "************Deleting local GIT files" + rm -Rf $1 +fi + + +# See if it exists on the Gitea host +REPO_NAME=$1 +RESPONSE=$(curl "$checkSSL" -s -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$REPO_NAME") +if [ "$RESPONSE" == "200" ]; then + echo "************Repository for $1 already exists!" + # If so, clone it (just want the README.md) + # And Clone it + cd $GITFiles + git clone "$GITEAHOST/$ORGGITEA/$1.git" + #GIT_SSL_NO_VERIFY=true git clone "$GITEAHOST/$ORGGITEA/$1.git" + NOREPO=0 +else + echo "************Repository for $1 does not exist." + NOREPO=1 +fi + +# then pull in the README +READMECONTENTS=""; +cd $GITFiles +if [ -d "$1" ] ; then + cd $GITFiles/$1 + # Retain the contents of the README.md (incase its been editted, and saving multiple GPT accesses) + if [ -f README.md ]; then + # Move it and remember its new (temp) name + READMECONTENTS=`mktemp` + mv -f README.md "$READMECONTENTS" + fi + echo "************Saved README: $READMECONTENTS" + cd $GITFiles + # Delete all files, links and directories + echo "************Deleting local GIT files" + rm -Rf $1 +fi + + +cd $GITFiles +#git init $QUIET $1 + +# Now delete the remote repo and re-create it and clone it in to local +curl "$checkSSL" -X 'DELETE' \ + "$GITEAHOST/api/v1/repos/$ORGGITEA/$1" \ + -H 'accept: application/json' \ + -H "Authorization: token $GITEAACCESSTOKEN" + +# and check if it is here. +RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$1") +if [ "$RESPONSE" == "200" ]; then + echo "************Repository for $1 exists!" + GOTREPO=0 +else + echo "************Repository for $1 does not exist" + GOTREPO=1 +fi + +# Work out if it is an SMEServer package or a 3rd party one +if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then + PACKAGETYPE="SMEServer Koozali developed" +else + PACKAGETYPE="3rd Party (Maintained by Koozali)" +fi + +# +# Use the API interface to create the remote repo +# +echo "************Creating repo for $1 $PACKAGETYPE" + +if [[ $GOTREPO -eq 1 ]] ; then + #Create it if it does not exist + JSONREPO=$(curl "$checkSSL" -X 'POST' "$GITEAHOST/api/v1/orgs/$ORGGITEA/repos" \ + -H 'accept: application/json' \ + -H "Authorization: token $GITEAACCESSTOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "auto_init": true, + "default_branch": "master", + "description": "'"$PACKAGETYPE git repo for $1 $2"'", + "gitignores": "", + "issue_labels": "Default", + "license": "Apache-2.0", + "name": "'"$1"'", + "private": false, + "readme": "Default", + "template": false, + "trust_model": "Default Trust Model" + }') + + echo $JSONREPO + + # and extract the http URL for it + HTTP_URL=`echo $JSONREPO | jq -r '.html_url'` + + echo $HTTP_URL + + if [[ -z "${HTTP_URL}" ]]; then + echo "ERROR*******Repo for $1 appears NOT to have been created " + echo "$JSONREQ" + exit 1 + fi +else + # Already there + HTTP_URL=$RemoteRepoURL + JSONREQ="$1 Already exists" +fi + +# And check it now exists +RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORGGITEA/$1") +if [ "$RESPONSE" == "200" ]; then + echo "************Repository for $1 exists" +else + echo "ERROR*******Repository for $1 has not been created ($JSONREQ)" + exit 1 +fi + +# Pull in the auto created or updated files +cd $GITFiles +#GIT_SSL_NO_VERIFY=true git clone "$GITEAHOST/$ORGGITEA/$1.git" +git clone "$GITEAHOST/$ORGGITEA/$1.git" + +if [ ! -d $GITFiles/$1 ] ; then + echo "ERROR*******Unable to access the new repo directory for $HTTP_URL $GITFiles/$1" + exit 1 +fi + +cd $CVSFiles/$1/$BASEORCONTRIB + +# Fully populate the directory (gets the tar file - which is not strictly in CVS) and creates the directory source tree +# applying the patches to the base in the tar file. +# Might need to replace this by the code from the makefile if we decide to move "make prep" to git moe likely we'll implement a new one "make gitprep". +make prep + +# Extract the directory name for the file tree (it will include the name of the module plus some version information) +# Same trick that we use in the makefile.common - taking the name and verison from the .spec file. +NAME=`ls *.spec|sed 's/\.spec//'` +VERSION=`rpm --queryformat '%{version}\n' --specfile $NAME.spec | head -n 1` +TREEPATH=$NAME-$VERSION +CODEDIR="root" +EXT=$(cat "$(basename $SPECFILE)" | grep -i ^Source | head -n 1 | sed -r 's/^[S|s]ource[0..9]?:\s?//' | xargs basename | sed -E 's/^([^.]*\.)(.*)$/\2/') +#echo $EXT $SPECFILE $(basename $SPECFILE) `pwd` +#exit 0 +ARCHIVEFILE=$(cat "$(basename $SPECFILE)" | grep ^Source | head -n 1 | sed -r 's/^[S|s]ource[0..9]?:\s?//' | xargs basename) +ARCHIVEFILE=$(rpm --queryformat "$ARCHIVEFILE\n" --specfile "$(basename $SPECFILE)" | head -n 1) + +#echo $ARCHIVEFILE $EXT $SPECFILE + +# Check that we got the filename (possible define in it not expanded?) +if [ ! -f $ARCHIVEFILE ]; then + # Try another tack + echo "************File $ARCHIVE does not exist - trying *.$EXT" + ARCHIVEFILE=`ls *.$EXT | head -n 1` + if [ ! -f $ARCHIVEFILE || -z $ARCHIVEFILE ]; then + echo "ERROR*******Can't identify archive file $ARCHIVEFILE $EXT `pwd`" + exit 1 + fi +fi +echo "************TreePATH:$TREEPATH and Archivefile:$ARCHIVEFILE `pwd`" + +# pull in all the files we want +cd $GITFiles/$1 + +# Add file indicating which release and contribs or base - this is matched by the makefile with the contents of the branch file in the common directory +# to give the details of the release (EL7/8/9/ etc) +# This assumes SME10 and contribs10 or sme10 directory names, later releases can update the value in the file and Makefile should still work. +echo $BASEORCONTRIB > contriborbase + +# Note that e-smith-backup base module has "Copying" not in all caps - seems to be the only one not in line. +BASESPECFILE=$(basename $SPECFILE) +for fpath in "$BASESPECFILE" "Makefile" +do + echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$fpath" + if [[ -f $CVSFiles/$1/$BASEORCONTRIB/$fpath ]] ; then + cp -R $CVSFiles/$1/$BASEORCONTRIB/$fpath . + else + echo "ERROR*******Expected to find $fpath in $CVSFiles/$1/$BASEORCONTRIB" + exit 1 + fi +done + +# and see if the spec file is not according to specification of .spec +# if not rename it +if [[ $SPECFILE != $ORIGSPECFILE ]] ; then + echo "************Renaming specfile from $(basename $SPECFILE) $(basename $ORIGSPECFILE)" + mv "$(basename $SPECFILE)" "$(basename $ORIGSPECFILE)" + SPECFILE=$ORIGSPECFILE +fi + +if [[ ! "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then + + # add in the patches (a 3rd party package) + # it turns out that not all patches end in .patch, so extract the file names out of the spec file as per sources below + PATCHS=$(grep -i -E '^patch[0-9]*:' $(basename $SPECFILE) | cut -d: -f2- | sed 's/ //g') + echo "$PATCHS" + + for fpath in $PATCHS; do + # Needs another step incase the extraction from the spec file contains spec variables + fpathfile=$(basename $fpath) + echo "$fpathfile" + translatefpath=$(rpm --queryformat "$fpathfile\n" --specfile $(basename $SPECFILE) | head -n 1) + echo "$translatefpath" + if [[ -f $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ]] ; then + echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$translatefpath"; + cp $CVSFiles/$1/$BASEORCONTRIB/$translatefpath . + else + echo "************Not found $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ( $fpathfile )- so not copied" + fi + done + + #for fpath in "*.patch" + #do + #echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$fpath" + #cp -R $CVSFiles/$1/$BASEORCONTRIB/$fpath . + #done + + # See if the spec file needs an extra buildrequires (nothing to do with git! - things have changed since SME10.1) + # if the spec file includes "BuildRequires: perl(ExtUtils::MakeMaker)" and NOT "BuildRequires: perl(ExtUtils::Manifest)" then add the latter + if grep -q "^BuildRequires:.*perl(ExtUtils::MakeMaker)" "$(basename $SPECFILE)" && ! grep -q "^BuildRequires:.*perl(ExtUtils::Manifest)" "$(basename $SPECFILE)"; then + sed -i '/^BuildRequires:.*perl(ExtUtils::MakeMaker)/a BuildRequires: perl(ExtUtils::Manifest)' "$(basename $SPECFILE)" + echo "************Added 'BuildRequires: perl(ExtUtils::Manifest)' to the spec file." + else + echo "************No action was taken. The spec file either does not include 'BuildRequires: perl(ExtUtils::MakeMaker)' or already includes 'BuildRequires: perl(ExtUtils::Manifest)'." + fi + + # and similar for another ExtUtils package - ParseXS + if grep -q "^BuildRequires:.*perl(ExtUtils::MakeMaker)" "$(basename $SPECFILE)" && ! grep -q "^BuildRequires:.*perl(ExtUtils::ParseXS)" "$(basename $SPECFILE)"; then + sed -i '/^BuildRequires:.*perl(ExtUtils::MakeMaker)/a BuildRequires: perl(ExtUtils::ParseXS)' "$(basename $SPECFILE)" + echo "************Added 'BuildRequires: perl(ExtUtils::ParseXS)' to the spec file." + else + echo "************No action was taken. The spec file either does not include 'BuildRequires: perl(ExtUtils::MakeMaker)' or already includes 'BuildRequires: perl(ExtUtils::ParseXS)'." + fi + + # and similar for another devel package - needs manifest + if grep -q "^BuildRequires:.*perl-devel" "$(basename $SPECFILE)" && ! grep -q "^BuildRequires:.*perl(ExtUtils::Manifest)" "$(basename $SPECFILE)"; then + sed -i '/^BuildRequires:.*perl-devel/a BuildRequires: perl(ExtUtils::Manifest)' "$(basename $SPECFILE)" + echo "************Added 'BuildRequires: perl(ExtUtils::Manifest)' to the spec file." + else + echo "************No action was taken. The spec file either does not include 'BuildRequires: perl-devel' or already includes 'BuildRequires: perl(ExtUtils::Manifest)'." + fi + + # Fix the rm in the pear packages - .filemap is no longer part of the download + if grep -q "^rm %{buildroot}/%{peardir}/.filemap" "$(basename $SPECFILE)"; then + sed -i 's/^rm *\%/rm -rf %/' "$(basename $SPECFILE)" + echo "************Added 'rm -rf %{buildroot}/%{peardir}/.filemap' to the spec file." + else + echo "************No action was taken. The spec file either does not include 'rm %{buildroot}/%{peardir}/.filemap' or already includes it" + fi + + +fi + +## and there may be extra files that maybe needed, these can be identified from the "Source" spec in the .spec files +SOURCES=$(grep -i -E '^Source[0-9]*:' $(basename $SPECFILE) | cut -d: -f2- | sed 's/ //g') +echo "$SOURCES" + +#SOURCE=$(rpm --queryformat '$SOURCE\n' --specfile $SPECFILE | head -n 1) +for fpath in $SOURCES; do + # Needs another step incase the extraction from the spec file contains spec variables + fpathfile=$(basename $fpath) + echo "$fpathfile" + translatefpath=$(rpm --queryformat "$fpathfile\n" --specfile $(basename $SPECFILE) | head -n 1) + echo "$translatefpath" + if [[ -f $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ]] ; then + echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/$translatefpath"; + cp $CVSFiles/$1/$BASEORCONTRIB/$translatefpath . + else + echo "************Not found $CVSFiles/$1/$BASEORCONTRIB/$translatefpath ( $fpathfile )- so not copied" + fi +done + +# Take out the logs and rpms from the git save +echo "*.rpm" >> .gitignore +echo "*.log" >> .gitignore +# and the backed up spec file from BogusDateBot.sh +echo "*spec-2023*" >>.gitignore + +cd $GITFiles/$1 +# See if the package is a 3rd party or one of ours - indicated by "smeserver" or "e-smith" in the name +if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then + # one of ours + echo "************Found SMEServer source for $NAME" + # Create the .gitignore file to ignore the archive + echo "*.$EXT" >> .gitignore + #and copy in the source tree + mkdir -p "$CODEDIR" + echo "************Copying $CVSFiles/$1/$BASEORCONTRIB/root `pwd`/$CODEDIR" + rsync -a --no-prune-empty-dirs "$CVSFiles/$1/$BASEORCONTRIB/$TREEPATH/root/" "`pwd`/$CODEDIR/" + # copy createlinks to base + if [ -e "$CVSFiles/$1/$BASEORCONTRIB/$TREEPATH/createlinks" ] ; then cp "$CVSFiles/$1/$BASEORCONTRIB/$TREEPATH/createlinks" "`pwd`/." ; fi + # copy anything else in the base directory of $TREEPATH into the additional directory + if [ ! -d "`pwd`/additional" ] ; then mkdir -p "`pwd`/additional" ; fi + rsync -av --exclude "createlinks" --exclude "root/" "$CVSFiles/$1/$BASEORCONTRIB/$TREEPATH/" "`pwd`/additional/" + # and plug a dummy into any empty directories + find "`pwd`/root"/* -type d -empty -exec touch {}/.gitignore \; +else + # Third party package + # copy over the archive file and make sure the extension is tracked by git-lfs + echo "************Found 3rd party package $NAME" + git lfs install + git lfs track "*.$EXT" + git add .gitattributes + #git config lfs.http://gitea.bjsystems.co.uk/SMEContribs/bugzilla.git/info/lfs.locksverify true + if [[ -f $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE ]] ; then + echo "************Copying archivefile $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE" + cp $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE . + else + echo "Unable to find archivefile: $CVSFiles/$1/$BASEORCONTRIB/$ARCHIVEFILE - nothing copied" + fi + cd $GITFiles/$1 +fi + +cd $GITFiles/$1 +if [ $ARCHIVEFILE != "$TREEPATH.tar.gz" ] ; then + echo "$ARCHIVEFILE" > archivefilename +fi + +# Take out any bogus dates: +/usr/bin/BogusDateBot.sh $(basename $SPECFILE) + +# If it is an SME package, take out the patches and then update the release. +if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then + # Now edit the spec file to remove all those patches and adjust the paths + sed -i '/^Patch/d; /^%patch/d' $(basename $SPECFILE) + # no paths need adjusting at this time! + # Bump the release and add changelog + change-log $(basename $SPECFILE) + # and edit in the reason + # sed - TBD - to look for "fix [SME: ]" + sed -i 's/fix \[SME: \]/Roll up patches and move to git repo \[SME: 12338\]/' $(basename $SPECFILE) + sed -i 's/ME MYSELF /cvs2git.sh aka Brian Read /' $(basename $SPECFILE) + # Add in the autochangelog and autorelease macros into the spec file + # Commented out for now until we have build systems that can install the rpmautospec package (CEntos, Rocky etc V9 only so far) +# sed -i 's/Release:.*/Release: %autorelease/' $(basename $SPECFILE) + # and add in the autochangelog if it is not already there +# sed -i '/^%changelog$/a %autochangelog' $(basename $SPECFILE) + # Note that the Readme.md is updated below once the repo has been created +fi + +## Clone the common directory - delete the contents first +#cd $GITFiles/$1 +#if [ -d common ] +#then + #find common -delete +#fi +#git clone "${cvs2git_COMMONREPO}" + +## Make sure common directory go created +#if [ ! -e $GITFiles/$1/common ]; then + #echo "ERROR*******No Common directory after git clone" + #exit 1 +#fi +## and delete the .git else it pushes back a link to this repo in the new repo +#cd $GITFiles/$1/common +#rm -fr ".git" +## and set the make-archive.sh executible - this needs to be done on any git clone - not sure how or where. +#chmod +x make-archive.sh + +# Re-factor the Common directory so that it is outside the repo being created. +# one Common directory for each Organisation +# +# See if it already exists +if [ ! -e $GITFiles/common ]; then + #Get it + cd $GITFiles + git clone "${cvs2git_COMMONREPO}" + if [ ! -e $GITFiles/common ]; then + echo "ERROR*******No Common Repo found - package $1" + exit 1 + fi +else + # Just refresh it + cd $GITFiles/common + git pull +fi + +# Now make sure that the make-archive is executible +chmod +x $GITFiles/common/make-archive.sh + +# Edit the Readme.md - try to add a link to the wiki for this package - only if it is an SME package +cd $GITFiles/$1 +echo "************READMECONTENTS:$READMECONTENTS" +PACKAGENAME=${NAME} +if [[ "$NAME" =~ ^(smeserver-|e-smith-).* ]]; then + # See if we have a previous contents of the README.md + if [[ "$NAME" =~ ^(smeserver-).* ]]; then + PACKAGENAME=${NAME#smeserver-} + fi + + # first find the relevant wiki entry + + MEDIAWIKI_SITE="https://wiki.koozali.org/api.php" + SEARCH_TERM="$PACKAGENAME" + + # Call the API to perform the search and store the JSON response in a variable + RESPONSE=$(curl -s "$MEDIAWIKI_SITE?action=query&format=json&list=search&srsearch=${SEARCH_TERM}&srprop=size%7Cwordcount%7Ctimestamp%7Csnippet&srlimit=10") + + # Use jq to extract the titles and pageids of the pages that match the search term (case-insensitive) + RESULTS=$(echo "$RESPONSE" | jq -r '.query.search[] | select(.title | ascii_downcase | contains("'"${SEARCH_TERM}"'" | ascii_downcase)) | .title, .pageid') + + # Loop through the results and construct the URL for each page + URLS=$(while read -r TITLE; do \ + read -r PAGEID; \ + URL="https://wiki.koozali.org/${TITLE}"; \ + URL=$(echo $URL | sed 's/ /_/g'); \ + echo "
${URL}"; \ + done <<< "${RESULTS}" ) + # and get the first non french (sorry JP!) + WIKILINK=$(while read -r TITLE; do \ + read -r PAGEID; \ + URL="https://wiki.koozali.org/${TITLE}"; \ + URL=$(echo $URL | sed 's/ /_/g'); \ + if [[ ! "$URL" =~ 'fr' ]] ; then echo "${URL}"; break; fi \ + done <<< "${RESULTS}" ) + + if [ "$READMECONTENTS" == "" ]; then + # nothing previous + # Copy across the Wiki links to the README + if [ ! "$URLS" = "" ]; then + echo "************The wiki page(s) for $PACKAGENAME exist." + echo -en "\n\n## Wiki\n$URLS" >> README.md + fi + + # And add bugzilla link + echo -en "\n\n## Bugzilla" >> README.md + echo -en "\nShow list of outstanding bugs: [here](https://bugs.koozali.org/buglist.cgi?component=$1&product=$PRODUCTBUGZILLA&query_format=advanced&limit=0&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&bug_status=CONFIRMED)" >> README.md + # Edit the header to add the logo + #KOOZALILOGO='' + KOOZALILOGO='' + if [[ "$NAME" =~ ^(smeserver-).* ]]; then + sed -i "s/^# smeserver/# $KOOZALILOGO smeserver/" README.md + else + sed -i "s/^# e-smith/# $KOOZALILOGO e-smith/" README.md + fi + # Good to extract some of the wiki here, and load it into the readme, + # OR I could ask GPTChat to write me paragraph on the contrib 8-)) + # Set the parameters for the request + if [[ "$NAME" =~ ^(smeserver-).* && "$2" == "contrib" ]]; then + prompt="Write me a paragraph on the $PACKAGENAME software contrib for smeserver" + else + prompt="Write me a paragraph about the software $PACKAGENAME" + fi + model="text-davinci-003" + temperature=0.8 + # Send the request to the API + response=$(curl -X POST https://api.openai.com/v1/engines/$model/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{"prompt": "'"$prompt"'", "temperature": 0.8, "max_tokens": 256}') + # Extract the text from the response + PARAGRAPH=`echo $response | jq -r '.choices[].text'` + # Add it to what was there already + echo "************Creating description for $1" + echo -e "\n\n## Description\n" >> README.md + echo -e "
*This description has been generated by an LLM AI system and cannot be relied on to be fully correct.*" >> README.md + echo -e "*Once it has been checked, then this comment will be deleted*" >> README.md + echo -e "
$PARAGRAPH" >> README.md + else + # overwrite with previous contents + echo "************Picked up previous README contents" + mv -f $READMECONTENTS ./README.md + fi +else + if [ "$READMECONTENTS" == "" ]; then + # Add in at least a description + prompt="Tell me what the software package $PACKAGENAME does?" + model="text-davinci-003" + temperature=0.8 + # Send the request to the API + response=$(curl -X POST https://api.openai.com/v1/engines/$model/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{"prompt": "'"$prompt"'", "temperature": 0.8, "max_tokens": 256}') + # Extract the text from the response + PARAGRAPH=`echo $response | jq -r '.choices[].text'` + # Add it to what was there already + echo "************Creating description for $1" + echo -e "\n\n## Description\n" >> README.md + echo -e "
*This description has been generated by an LLM AI system and cannot be relied on to be fully correct.*" >> README.md + echo -e "*Once it has been checked, then this comment will be deleted*" >> README.md + echo -e "
$PARAGRAPH" >> README.md + else + # overwrite with previous contents + echo "************Picked up previous README contents" + mv -f $READMECONTENTS ./README.md + fi +fi + +# Tell repo to use koozali.org bugzilla and wiki - this does not work on CREATE currently - 19Apr2023 +if [[ -z $WIKILINK ]] ; then + WIKILINK="https://wiki.koozali.org" +fi + +# and then update the repo parameters +RESPONSE=$(curl -X 'PATCH' "$GITEAHOST/api/v1/repos/$ORGGITEA/$REPO_NAME" \ + -H 'accept: application/json' \ + -H "Authorization: token $GITEAACCESSTOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "external_tracker": { + "external_tracker_format": "https://bugs.koozali.org/show_bug.cgi?id={index}", + "external_tracker_style": "numeric", + "external_tracker_url": "'"https://bugs.koozali.org/buglist.cgi?component=$REPO_NAME&product=$PRODUCTBUGZILLA"'" + }, + "external_wiki": { + "external_wiki_url": "'"$WIKILINK"'" + } + } + ') +# should check response here. +echo $RESPONSE +#exit 0 + +cd $GITFiles/$1 +# stage and commit all the files created locally +git add -A +echo "$1 git status" >>gitstatus.log +git status >>gitstatus.log +git status +COMMENT="initial commit of file from CVS for $1 on `date`" +git commit -m "$COMMENT" $QUIET + +# push the converted git repository to the remote +git remote remove origin +git remote add origin "$GITEAHOST/$ORGGITEA/$1.git" +#GIT_SSL_NO_VERIFY=true git push --force -u $QUIET origin master +git push --force -u $QUIET origin master + +# Now create the version and release tag for these files +RELEASE=`rpm --queryformat '%{release}\n' --specfile $NAME.spec | head -n 1` +TAG=$VERSION-$RELEASE +git tag -a $TAG -m "$COMMENT" +git push origin $TAG + +echo "Created $1 Repo: $HTTP_URL - Comment:$COMMENT tag:$TAG" +exit 0 + + diff --git a/git-get-repo-and-build.sh b/git-get-repo-and-build.sh new file mode 100755 index 0000000..92a2582 --- /dev/null +++ b/git-get-repo-and-build.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# clone the repo into the local relevant directory and run mockbuild on it +# monitor results and log +# +inifilename=$(echo ~)"/.smegit/config" +if [ -z $inifilename ] ; then + # Not here, look a bit further up + echo "No ini file found $inifiename" + echo "get-repo-and-build.sh []" + exit 1 +fi +while read -r line || [[ -n "$line" ]]; do + if [[ $line =~ ^\[.*\]$ ]] + then + section=${line#*[} + section=${section%]*} + else + if [[ $line =~ ^[^#]*= ]] + then + key=${line%=*} + value=${line#*=} + declare "${section}_${key}=$value" + fi + fi +done < "$inifilename" + +if [[ -z $1 && -z $2 ]] ; then + echo "get-repo-and-build.sh []" + exit 0 +fi + +DEBUG= +if [ ${smegit_DEBUG} == "true" ] ; then DEBUG=true ; fi +if [ $DEBUG ] ; then echo "found ini file: $inifilename" ; fi + +curlsilent="-s" +if [ $DEBUG ] ; then curlsilent= ; fi + +GITEAHOST=${remote_GITEAHOST} +ACCESSTOKEN=${remote_GITEAACCESSTOKEN} +if [ $3 ] ; then + if [ $3 == "local" ] ; then + GITEAHOST=${local_GITEAHOST} + ACCESSTOKEN=${local_GITEAACCESSTOKEN} + fi +fi +ORG_NAME="$1" +REPO_NAME="$2" +WORKDIR=$(echo ~)${smegit_WORKDIR} +GITFiles=$WORKDIR/GITFiles/$ORG_NAME +REPOURL="$GITEAHOST/$ORG_NAME/$REPO_NAME.git" + +if [ $DEBUG ] ; then echo "getting $REPO_NAME from $ORG_NAME on $GITEAHOST" ; fi + +mkdir -p $GITFiles + +touch "$GITFiles/mockbuilds.log" + +if [ $DEBUG ] ; then echo $GITFiles ; fi + +# one Common directory for each Organisation +# +# See if it already exists +if [ ! -e $GITFiles/common ]; then + #Get it + cd $GITFiles + git clone "${smegit_COMMONREPO}" + if [ ! -e $GITFiles/common ]; then + echo "ERROR*******No Common Repo found - package $1" + exit 1 + fi +else + # Just refresh it + cd $GITFiles/common + git pull +fi + +# Now make sure that the make-archive is executible +chmod +x $GITFiles/common/make-archive.sh + +# See if repo exits in git +RESPONSE=$(curl $curlsilent -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORG_NAME/$REPO_NAME") +if [ "$RESPONSE" == "200" ]; then + echo "Repository for $1/$2 exists!" +else + echo "Repository for $1/$2 does not exist" + logname="git-cvs2git-create-$2-$1-$(date +'%FT%H%M').log" + while true; do + read -p "Do you wish to run git-cvs2git?(y/n) " yn + case $yn in + [Yy]* ) git-cvs2git.sh "$REPO_NAME" "$ORG_NAME" > $logname; break;; + [Nn]* ) exit;; + * ) echo "Please answer yes or no.";; + esac + done + # And check it is now there + RESPONSE=$(curl $curlsilent -o /dev/null -w "%{http_code}" "$GITEAHOST/api/v1/repos/$ORG_NAME/$REPO_NAME") + if [ "$RESPONSE" == "200" ]; then + echo "Repository for $1/$2 created sucessfully!" + else + echo "git-cvs2git.sh was unable to create $1/$2 sucessfully" + exit 1 + fi +fi +# Ok, we have a repo!! +cd $GITFiles +echo $GITFiles +# Delete it if these already +if [[ -d $REPO_NAME ]] ; then + # Should delete everything.... + cd $GITFiles/$REPO_NAME + echo "Deleting all files in $GITFiles/$REPO_NAME" + rm -rf "$REPO_NAME" +fi +cd $GITFiles +git clone "$REPOURL" + + +cd $GITFiles +if [[ ! -d $REPO_NAME ]] ; then + echo "git clone has not created the directory" + exit 1 +fi +cd $GITFiles/$REPO_NAME +# and run mockbuild +logname="mockbuild-$2-$1-$(date +'%FT%H%M').log" +if make mockbuild 2>&1 > $logname ; then + echo "Looks like mockbuild worked" + echo "$(date +'%FT%H%M') Mockbuild worked for $1/$2 " >> $GITFiles/mockbuilds.log +else + echo "Mockbuild failed" + echo "$(date +'%FT%H%M') Mockbuild failed for $1/$2 " >> $GITFiles/mockbuilds.log + exit 1 +fi + + + + + + + + + diff --git a/git-mirror-org.sh b/git-mirror-org.sh new file mode 100755 index 0000000..0f93662 --- /dev/null +++ b/git-mirror-org.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Migrate (mirror) a remote repository +# +# migrate-repo.sh [] +# +inifilename=$(echo ~)"/.smegit/config" +if [ -z $inifilename ] ; then + # Not here, look a bit further up + echo "No ini file found $inifilename" + echo "git-mirror-org.sh []" + exit 1 +fi + +while read -r line || [[ -n "$line" ]]; do + if [[ $line =~ ^\[.*\]$ ]] + then + section=${line#*[} + section=${section%]*} + else + if [[ $line =~ ^[^#]*= ]] + then + key=${line%=*} + value=${line#*=} + declare "${section}_${key}=$value" + fi + fi +done < "$inifilename" + +if [[ -z $1 ]] ; then + echo "migrate-org.sh []" + echo " is optional and will default to the gitea user" + exit 0 +fi +DEBUG= +if [ ${smegit_DEBUG} == true ] ; then DEBUG=true ; fi +if [ $DEBUG ] ; then echo "found ini file: $inifilename" ; fi + +curlsilent="-s" +if [ $DEBUG ] ; then curlsilent= ; fi + +REMOTEACCESSTOKEN=${remote_GITEAACCESSTOKEN} +LOCALACCESSTOKEN=${local_GITEAACCESSTOKEN} +REMOTEGITEAHOST=${remote_GITEAHOST} +LOCALGITEAHOST=${local_GITEAHOST} +TARGETORG= +if [ $2 ] ; then TARGETORG=$2 ; fi + +# check that the target organisation exists locally +RESPONSE=$(curl $curlsilent -o /dev/null -w "%{http_code}" $LOCALGITEAHOST/$TARGETORG) +if [[ "$RESPONSE" != "200" ]] ; then + echo "$TARGETORG does not exist on $LOCALGITEAHOST" + exit 1 +fi + +# get a list of repositories in the source organisation +RESPONSE=$(curl $curlsilent -X 'GET' \ + "$REMOTEGITEAHOST/api/v1/orgs/$1/repos?limit=1000?per_page=200" \ + -H 'accept: application/json' \ + -H "Authorization: token $REMOTEACCESSTOKEN" +) + +echo $RESPONSE | grep -oP '(?<="name":").+?(?=")' | while read repo; do + echo "git-mirror-repo.sh ${repo} $1 $TARGETORG" +done diff --git a/git-mirror-repo.sh b/git-mirror-repo.sh new file mode 100755 index 0000000..91e3b4a --- /dev/null +++ b/git-mirror-repo.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Migrate (mirror) a remote repository +# +# migrate-repo.sh [] +# +inifilename=$(echo ~)"/.smegit/config" +if [ -z $inifilename ] ; then + # Not here + echo "No ini file found $inifilename" + echo "git_mirror-repo.sh []" + exit 1 +fi + +while read -r line || [[ -n "$line" ]]; do + if [[ $line =~ ^\[.*\]$ ]] + then + section=${line#*[} + section=${section%]*} + else + if [[ $line =~ ^[^#]*= ]] + then + key=${line%=*} + value=${line#*=} + declare "${section}_${key}=$value" + fi + fi +done < $inifilename + +if [[ -z $1 ]] ; then + echo "git_migrate-repo.sh []" + echo "will migrate repo to ${local_USER} unless specified" + exit 0 +fi +DEBUG= +if [ ${smegit_DEBUG} == true ] ; then DEBUG=true ; fi +if [ $DEBUG ] ; then echo "found ini file: $inifilename" ; fi + +curlsilent="-s" +if [ $DEBUG ] ; then curlsilent= ; fi + +REMOTEUser=${remote_USER} +REMOTEACCESSTOKEN=${remote_GITEAACCESSTOKEN} +LOCALACCESSTOKEN=${local_GITEAACCESSTOKEN} +REMOTEGITEAHOST=${remote_GITEAHOST} +CLONEADDRESS=$REMOTEGITEAHOST/$2/$1.git +LOCALGITEAHOST=${local_GITEAHOST} +NEWORG=${local_USER} +if [ $3 ] ; then NEWORG=$3 ; fi + +# check that the NEWORG exists +RESPONSE=$(curl $curlsilent -o /dev/null -w "%{http_code}" $LOCALGITEAHOST/$NEWORG) +if [[ "$RESPONSE" != "200" ]] ; then + echo "$NEWORG does not exist on $LOCALGITEAHOST" + exit 1 +fi + +# And check if this repo already exists on the target +RESPONSE=$(curl $curlsilent -o /dev/null -w "%{http_code}" $LOCALGITEAHOST/$NEWORG/$1) +if [ "$RESPONSE" == "200" ] ; then + echo "Repository $NEWORG/$1 already exists on $LOCALGITEAHOST - cannot migrate" + exit 1 +fi + +if [ $DEBUG ] ; then echo "Mirroring $2/$1 as $NEWORG/$1 on $LOCALGITEAHOST" ; fi +RESPONSE=$(curl $curlsilent -k -X 'POST' \ + "$LOCALGITEAHOST/api/v1/repos/migrate" \ + -H 'accept: application/json' \ + -H "Authorization: token $LOCALACCESSTOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "auth_token": "'"$REMOTEACCESSTOKEN"'", + "clone_addr": "'"$CLONEADDRESS"'", + "issues": true, + "labels": true, + "lfs": true, + "milestones": true, + "mirror": true, + "private": false, + "pull_requests": true, + "releases": true, + "repo_name": "'"$1"'", + "repo_owner": "'"$NEWORG"'", + "service": "git", + "uid": 0, + "wiki": true +}') + +if [ $DEBUG ] ; then echo $? $RESPONSE ; fi + +# And check if this repo is there +if [ $DEBUG ] ; then echo "checking that $NEWORG/$1 exists" ; fi +RESPONSE=$(curl $curlsilent -o /dev/null -w "%{http_code}" $LOCALGITEAHOST/$NEWORG/$1) +if [ "$RESPONSE" == "200" ] ; then + echo "Repository $NEWORG/$1 has been created and mirrors $2/$1" +else + echo "migrate-repo was unable to migrate $2/$1 sucessfully" + exit 1 +fi +