[fe30c61] | 1 | #!/bin/bash
|
---|
| 2 |
|
---|
| 3 | # $Id$
|
---|
| 4 |
|
---|
| 5 |
|
---|
| 6 | #----------------------------#
|
---|
| 7 | get_sources() { # Download file, write name to MISSING_FILES.DMP if an error
|
---|
| 8 | #----------------------------#
|
---|
| 9 | local saveIFS=$IFS
|
---|
[de63126] | 10 | local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE WGETPARAM
|
---|
| 11 |
|
---|
| 12 | WGETPARAM=""
|
---|
| 13 | if [[ "${RETRYSRCDOWNLOAD}" = "y" ]] ; then
|
---|
| 14 | WGETPARAM+="--retry-connrefused"
|
---|
| 15 | fi
|
---|
| 16 | WGETPARAM+=" --tries ${RETRYDOWNLOADCNT}"
|
---|
| 17 | WGETPARAM+=" --timeout ${DOWNLOADTIMEOUT}"
|
---|
[fe30c61] | 18 |
|
---|
| 19 | # Test if the packages must be downloaded
|
---|
| 20 | [ ! "$GETPKG" = "y" ] && return
|
---|
| 21 |
|
---|
| 22 | gs_wrt_message(){
|
---|
| 23 | echo "${RED}$1${OFF}"
|
---|
| 24 | echo "$1" >> MISSING_FILES.DMP
|
---|
| 25 | }
|
---|
| 26 | # Housekeeping
|
---|
| 27 | [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
|
---|
| 28 | cd $BUILDDIR/sources
|
---|
[bb4b5c3] | 29 | # If using CLFS, /sources is writable by all, but with sticky bit,
|
---|
| 30 | # and user does not hold MD5SUMS nor the other files, so use sudo
|
---|
| 31 | [[ -f MD5SUMS ]] && sudo rm MD5SUMS
|
---|
| 32 | [[ -f MISSING_FILES.DMP ]] && sudo rm MISSING_FILES.DMP
|
---|
| 33 | [[ -f urls.lst ]] && sudo rm urls.lst
|
---|
[fe30c61] | 34 |
|
---|
| 35 | # Generate URLs file
|
---|
| 36 | create_urls
|
---|
| 37 |
|
---|
| 38 | IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
|
---|
| 39 | for line in `cat urls.lst`; do
|
---|
| 40 | IFS=$saveIFS # Restore the system defaults
|
---|
| 41 |
|
---|
| 42 | # Skip some packages if they aren't needed
|
---|
| 43 | case $line in
|
---|
[de84c00] | 44 | */tree* | */gcc-testsuite* ) #unneeded now?
|
---|
[fe30c61] | 45 | [[ "$TEST" = "0" ]] && continue
|
---|
| 46 | ;;
|
---|
[7072e1f] | 47 | */vim-*-lang* )
|
---|
[fe30c61] | 48 | [[ "$VIMLANG" = "0" ]] && continue
|
---|
| 49 | ;;
|
---|
| 50 | esac
|
---|
| 51 |
|
---|
| 52 | # Locations
|
---|
| 53 | URL1=`echo $line | cut -d" " -f2` # Preferred URL
|
---|
| 54 | URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL
|
---|
| 55 | FILE=`basename $URL1` # File name
|
---|
| 56 | BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value
|
---|
| 57 |
|
---|
| 58 | # Validation pair
|
---|
| 59 | MD5="$BOOKMD5 $FILE"
|
---|
| 60 | HAVEMD5=1
|
---|
| 61 |
|
---|
| 62 | set -e
|
---|
| 63 | # If the file exists in the archive copy it to the
|
---|
| 64 | # $BUILDDIR/sources dir. MD5SUM will be validated later.
|
---|
| 65 | if [ ! -z ${SRC_ARCHIVE} ] &&
|
---|
| 66 | [ -d ${SRC_ARCHIVE} ] &&
|
---|
| 67 | [ -f ${SRC_ARCHIVE}/$FILE ]; then
|
---|
| 68 | cp ${SRC_ARCHIVE}/$FILE .
|
---|
| 69 | echo "$FILE: -- copied from $SRC_ARCHIVE"
|
---|
| 70 | fromARCHIVE=1
|
---|
| 71 | else
|
---|
| 72 | fromARCHIVE=0
|
---|
| 73 | # If the file does not exist yet in /sources download a fresh one
|
---|
| 74 | if [ ! -f $FILE ] ; then
|
---|
[9d55d85] | 75 | if [[ ! ("$SRC_ARCHIVE" = "") ]] ; then
|
---|
[7072e1f] | 76 | echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE} or ${BUILDDIR}/sources${OFF}"
|
---|
[9d55d85] | 77 | else
|
---|
[7072e1f] | 78 | echo "${BOLD}${YELLOW}$FILE: not found in ${BUILDDIR}/sources${OFF}"
|
---|
[9d55d85] | 79 | fi
|
---|
[de63126] | 80 | if ! wget $URL1 $WGETPARAM && ! wget $URL2 $WGETPARAM ; then
|
---|
[fe30c61] | 81 | gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
|
---|
| 82 | continue
|
---|
| 83 | fi
|
---|
[7072e1f] | 84 | else
|
---|
| 85 | echo "${BOLD}${YELLOW}$FILE: using cached file in ${BUILDDIR}/sources${OFF}"
|
---|
[fe30c61] | 86 | fi
|
---|
| 87 | fi
|
---|
| 88 |
|
---|
[5210ae1] | 89 | # Deal with udev and bootscripts m5sum issue
|
---|
[0700131] | 90 | [[ $BOOKMD5 = "BOOTSCRIPTS-MD5SUM" ]] && continue
|
---|
| 91 | [[ $BOOKMD5 = "UDEV-MD5SUM" ]] && continue
|
---|
| 92 | [[ $BOOKMD5 = "LFS-NETSCRIPTS-MD5SUM" ]] && continue
|
---|
[5210ae1] | 93 |
|
---|
[fe30c61] | 94 | # IF the md5sum does not match the existing files
|
---|
| 95 | if ! echo "$MD5" | md5sum -c - >/dev/null ; then
|
---|
| 96 | [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
|
---|
| 97 | [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
|
---|
| 98 | # Remove the old file and download a new one
|
---|
| 99 | rm -fv $FILE
|
---|
| 100 | # Force storage in SRC_ARCHIVE
|
---|
| 101 | fromARCHIVE=0;
|
---|
| 102 | # Try to retrieve again the file. Servers in reverse order.
|
---|
[de63126] | 103 | if ! wget $URL2 $WGETPARAM && ! wget $URL1 $WGETPARAM ; then
|
---|
[fe30c61] | 104 | gs_wrt_message "$FILE not found on the servers.. SKIPPING"
|
---|
| 105 | continue
|
---|
| 106 | fi
|
---|
| 107 | fi
|
---|
| 108 |
|
---|
| 109 | # Validate the MD5SUM one last time
|
---|
| 110 | if ! echo "$MD5" | md5sum -c - >/dev/null ; then
|
---|
| 111 | gs_wrt_message "$FILE does not match MD5SUMS value"
|
---|
| 112 | # Force generation of MD5SUM
|
---|
| 113 | HAVEMD5=0
|
---|
| 114 | fi
|
---|
| 115 |
|
---|
| 116 | # Generate a fresh MD5SUM for this file
|
---|
| 117 | if [[ "$HAVEMD5" = "0" ]] ; then
|
---|
| 118 | echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
|
---|
| 119 | echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP
|
---|
| 120 | fi
|
---|
| 121 |
|
---|
| 122 | # Good or bad we write the original md5sum to a file
|
---|
[93fd2d0] | 123 | echo "$MD5" >> MD5SUMS
|
---|
[fe30c61] | 124 |
|
---|
| 125 | # Copy the freshly downloaded file
|
---|
| 126 | # to the source archive.
|
---|
| 127 | if [ ! -z ${SRC_ARCHIVE} ] &&
|
---|
| 128 | [ -d ${SRC_ARCHIVE} ] &&
|
---|
| 129 | [ -w ${SRC_ARCHIVE} ] &&
|
---|
| 130 | [ "$fromARCHIVE" = "0" ] ; then
|
---|
| 131 | echo "Storing file:<$FILE> in the package archive"
|
---|
| 132 | cp -f $FILE ${SRC_ARCHIVE}
|
---|
| 133 | fi
|
---|
| 134 |
|
---|
| 135 | done
|
---|
| 136 |
|
---|
| 137 | if [[ -s MISSING_FILES.DMP ]]; then
|
---|
| 138 | echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
|
---|
| 139 | # Do not allow the automatic execution of the Makefile.
|
---|
| 140 | echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
|
---|
| 141 | RUNMAKE="n"
|
---|
| 142 | fi
|
---|
| 143 | }
|
---|
| 144 |
|
---|
| 145 | #----------------------------#
|
---|
| 146 | create_urls() { #
|
---|
| 147 | #----------------------------#
|
---|
| 148 | cd $JHALFSDIR
|
---|
| 149 |
|
---|
| 150 | case ${PROGNAME} in
|
---|
[809a821] | 151 | clfs*)
|
---|
[fe30c61] | 152 | echo -n "Creating CLFS <${ARCH}> specific URLs file"
|
---|
| 153 | xsltproc --nonet --xinclude \
|
---|
| 154 | --stringparam server $SERVER \
|
---|
[a9ca77f] | 155 | --stringparam family clfs \
|
---|
[fe30c61] | 156 | -o $BUILDDIR/sources/urls.lst urls.xsl \
|
---|
[809a821] | 157 | $BOOK/BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
|
---|
[fe30c61] | 158 | echo " ...OK"
|
---|
| 159 | ;;
|
---|
| 160 | hlfs)
|
---|
[139c34e] | 161 | echo -n "Creating HLFS <${MODEL}> + <${KERNEL}> specific URLs file"
|
---|
[fe30c61] | 162 | xsltproc --nonet --xinclude \
|
---|
| 163 | --stringparam server $SERVER \
|
---|
[a9ca77f] | 164 | --stringparam family lfs \
|
---|
[fe30c61] | 165 | --stringparam model $MODEL \
|
---|
[139c34e] | 166 | --stringparam kernel $KERNEL \
|
---|
[fe30c61] | 167 | -o $BUILDDIR/sources/urls.lst urls.xsl \
|
---|
| 168 | $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1
|
---|
| 169 | echo " ...OK"
|
---|
| 170 | ;;
|
---|
| 171 | lfs)
|
---|
| 172 | echo -n "Creating LFS specific URLs file"
|
---|
| 173 | xsltproc --nonet --xinclude \
|
---|
| 174 | --stringparam server $SERVER \
|
---|
[a9ca77f] | 175 | --stringparam family lfs \
|
---|
[7072e1f] | 176 | --stringparam pkgmngt $PKGMNGT \
|
---|
[fe30c61] | 177 | -o ../sources/urls.lst urls.xsl \
|
---|
| 178 | $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1
|
---|
| 179 | echo " ...OK"
|
---|
| 180 | ;;
|
---|
| 181 | esac
|
---|
| 182 |
|
---|
| 183 | cd $BUILDDIR/sources
|
---|
| 184 |
|
---|
| 185 | if [[ "${CUSTOM_TOOLS}" = "y" ]]; then
|
---|
| 186 | add_CustomToolsURLS
|
---|
| 187 | fi
|
---|
| 188 |
|
---|
| 189 | }
|
---|
| 190 |
|
---|