source: common/libs/func_download_pkgs@ e5fe832

#!/bin/bash

# $Id$


#----------------------------#
get_sources() {              # Download file, write name to MISSING_FILES.DMP if an error
#----------------------------#
  local saveIFS=$IFS
  local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE

    # Test if the packages must be downloaded
  [ ! "$GETPKG" = "y" ] && return

  gs_wrt_message(){
    echo "${RED}$1${OFF}"
    echo "$1" >> MISSING_FILES.DMP
  }
    # Housekeeping
  [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
  cd $BUILDDIR/sources
  [[ -f MD5SUMS ]] && rm MD5SUMS
  [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP
  [[ -f urls.lst ]] && rm urls.lst

    # Generate URLs file
  create_urls

  IFS=$'\x0A'  # Modify the 'internal field separator' to break on 'LF' only
  for line in `cat urls.lst`; do
    IFS=$saveIFS  # Restore the system defaults

      # Skip some packages if they aren't needed
    case $line in
        */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* )
            [[ "$TEST" = "0" ]] && continue
          ;;
        */vim-*-lang* )
            [[ "$VIMLANG" = "0" ]] && continue
          ;;
        *linux/linux-* )
            [[ -z "$CONFIG" ]] && [[ -z "$BOOT_CONFIG" ]] && \
            [[ "$GETKERNEL" = "n" ]] && continue
          ;;
    esac

      # Locations
    URL1=`echo $line | cut -d" " -f2` # Preferred URL
    URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL
    FILE=`basename $URL1` # File name
    BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value

      # Validation pair
    MD5="$BOOKMD5  $FILE"
    HAVEMD5=1

    set -e
      # If the file exists in the archive copy it to the
      # $BUILDDIR/sources dir. MD5SUM will be validated later.
    if [ ! -z ${SRC_ARCHIVE} ] &&
       [   -d ${SRC_ARCHIVE} ] &&
       [   -f ${SRC_ARCHIVE}/$FILE ]; then
      cp ${SRC_ARCHIVE}/$FILE .
      echo "$FILE: -- copied from $SRC_ARCHIVE"
      fromARCHIVE=1
    else
      echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}"
      fromARCHIVE=0
      # If the file does not exist yet in /sources download a fresh one
      if [ ! -f $FILE ] ; then
        if ! wget $URL1 && ! wget $URL2 ; then
          gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
          continue
        fi
      fi
    fi

      # IF the md5sum does not match the existing files
    if ! echo "$MD5" | md5sum -c - >/dev/null ; then
      [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
      [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
        # Remove the old file and download a new one
      rm -fv $FILE
        # Force storage in SRC_ARCHIVE
      fromARCHIVE=0;
        # Try to retrieve again the file. Servers in reverse order.
      if ! wget $URL2 && ! wget $URL1 ; then
        gs_wrt_message "$FILE not found on the servers.. SKIPPING"
        continue
      fi
    fi

      # Validate the MD5SUM one last time
    if ! echo "$MD5" | md5sum -c - >/dev/null ; then
      gs_wrt_message "$FILE does not match MD5SUMS value"
      # Force generation of MD5SUM
      HAVEMD5=0
    fi

      # Generate a fresh MD5SUM for this file
    if [[ "$HAVEMD5" = "0" ]] ; then
      echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
      echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP
    fi

      # Good or bad we write the original md5sum to a file
    echo "$MD5" >> MD5SUMS

     # Copy the freshly downloaded file
     # to the source archive.
   if [ ! -z ${SRC_ARCHIVE}  ] &&
      [   -d ${SRC_ARCHIVE}  ] &&
      [   -w ${SRC_ARCHIVE}  ] &&
      [ "$fromARCHIVE" = "0" ] ; then
        echo "Storing file:<$FILE> in the package archive"
        cp -f $FILE ${SRC_ARCHIVE}
   fi

  done

  if [[ -s MISSING_FILES.DMP ]]; then
    echo  -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
      # Do not allow the automatic execution of the Makefile.
    echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
    RUNMAKE="n"
  fi
}

#----------------------------#
create_urls() {              #
#----------------------------#
  cd $JHALFSDIR

  case ${PROGNAME} in
    clfs)
        echo -n "Creating CLFS <${ARCH}> specific URLs file"
        xsltproc --nonet --xinclude \
                 --stringparam server $SERVER \
                 --stringparam family clfs \
                 -o $BUILDDIR/sources/urls.lst $CODE_DIR/XSL/urls.xsl \
                 $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
      ;;
    clfs2)
        echo -n "Creating CLFS2 <${ARCH}> specific URLs file"
        xsltproc --nonet --xinclude \
                 --stringparam server $SERVER \
                 --stringparam family clfs \
                 -o $BUILDDIR/sources/urls.lst $CODE_DIR/XSL/urls.xsl \
                 $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
      ;;
    clfs3)
        echo -n "Creating CLFS3 <${ARCH}> specific URLs file"
        xsltproc --nonet --xinclude \
                 --stringparam server $SERVER \
                 --stringparam family clfs \
                 -o $BUILDDIR/sources/urls.lst $CODE_DIR/XSL/urls.xsl \
                 $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
      ;;
    hlfs)
        echo -n "Creating HLFS <${MODEL}> + <${KERNEL}> specific URLs file"
        xsltproc --nonet --xinclude \
                 --stringparam server $SERVER \
                 --stringparam family lfs \
                 --stringparam model $MODEL \
                 --stringparam kernel $KERNEL \
                 -o $BUILDDIR/sources/urls.lst $CODE_DIR/XSL/urls.xsl \
                 $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1
      ;;
    lfs)
        echo -n "Creating LFS specific URLs file"
        xsltproc --nonet --xinclude \
                 --stringparam server $SERVER \
                 --stringparam family lfs \
                 -o ../sources/urls.lst $CODE_DIR/XSL/urls.xsl \
                 $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1
      ;;
  esac

  add_extra_urls

  echo " ...OK"

  cd $BUILDDIR/sources

}

#----------------------------# Add download URLs found in user added scripts,
add_extra_urls() {           # blfs-tool-deps scripts, and custom-tools scripts
#----------------------------#
  local saveIFS=$IFS
  local IFS file line package tarball download md5sum patchurl patchname patchmd5

  local BLFS_SERVER="${SERVER}/pub/blfs/conglomeration/"

  for file in ${PROGNAME}-commands/*/* ; do
    # Grab the package name, tarball, download URL and MD5SUM
    package=`grep "^PACKAGE=" ${file} | sed -e 's@PACKAGE=@@'`
    tarball=`grep "^TARBALL=" ${file} | sed -e 's@TARBALL=@@'`
    download=`grep "^DOWNLOAD=" ${file} | sed -e 's@DOWNLOAD=@@'`
    md5sum=`grep "^MD5SUM=" ${file} | sed -e 's@MD5SUM=@@'`

    # Be sure that we have a download URL and MD5SUM value
    if [ -n "${download}" ] && [ -n "${md5sum}" ] ; then
      # Write the package entry
      echo "${download} ${BLFS_SERVER}${package}/${tarball} ${md5sum}" >> ../sources/urls.lst
    fi

    # Handle the patches, if any
    IFS=$'\x0A'
    for line in `grep "^PATCH=" ${file}` ; do
      IFS=$saveIFS
      patchurl=`echo ${line} | sed -e 's@PATCH="@@' -e 's@ .*@@'`
      patchname=${patchurl##*/}
      patchmd5=`echo ${line} | sed -e 's@.* @@' -e 's@"@@'`

      # Write the patch entry
      echo "${patchurl} ${BLFS_SERVER}${package}/${patchname} ${patchmd5}" >> ../sources/urls.lst
    done
  done
}