Changeset 93135fd

Timestamp:

05/03/2006 12:25:38 AM (18 years ago)

Author:

George Boudreau <georgeb@…>

Branches:

1.0, 2.3, 2.3.x, 2.4, ablfs, ablfs-more, legacy, new_features, trunk

Children:

511923a

Parents:

47e524f

Message:

Rewrote the package download function get_sources

File:

• common/common-functions (modified) (1 diff)

common/common-functions

@@ -628 +628 @@
 }
 
-
 #----------------------------#
 get_sources() {              # Download file, write name to MISSING_FILES.DMP if an error
+#----------------------------#
+  local saveIFS=$IFS
+  local IFS line URL1 URL2 FILE MD5 HAVEMD5 fromARCHIVE
+
+    # Test if the packages must be downloaded
+  [ ! "$GETPKG" = "1" ] && return
+
+  gs_wrt_message(){
+    echo "${RED}$1${OFF}"
+    echo "$1" >> MISSING_FILES.DMP    
+  }  
+    # Housekeeping
+  [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
+  cd $BUILDDIR/sources
+  [[ -f MD5SUMS ]] && rm MD5SUMS
+  [[ -f MD5SUMS-$VERSION ]] && rm MD5SUMS-$VERSION
+  [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP
+  [[ -f urls.lst ]] && rm urls.lst
+
+    # Download a fresh MD5SUMS file
+  wget $SERVER/pub/lfs/conglomeration/MD5SUMS
+
+    # Generate URLs file
+  create_urls
+
+  IFS=$'\x0A'  # Modify the 'internal field separator' to break on 'LF' only
+  for line in `cat urls.lst`; do
+    IFS=$saveIFS  # Restore the system defaults
+
+      # Skip some packages if they aren't needed
+    case $line in
+        */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* )
+            [[ "$TEST" = "0" ]] && continue
+          ;;
+        */vim-*-lang* )
+            [[ "$VIMLANG" = "0" ]] && continue
+          ;;
+    esac
+
+      # Locations
+    URL1=`echo $line | cut -d" " -f1` # Upstream URL
+    URL2=`echo $line | cut -d" " -f2` # Fallback URL
+    FILE=`basename $URL2` # File name
+
+      # Find the md5 sum for this package.
+    set +e
+    HAVEMD5=1  # Always assume we have a MD5SUM
+    MD5=`grep "  $FILE" MD5SUMS`
+    if [ $? -ne 0 ]; then
+        set -e
+        gs_wrt_message "$FILE not found in MD5SUMS"
+          # IMPORTANT:: There is no MD5SUM for this file.
+        HAVEMD5=0
+    fi
+
+    set -e
+      # If the file exists in the archive copy it to the 
+      # $BUILDDIR/sources dir. MD5SUM will be validated later.
+    if [ ! -z ${SRC_ARCHIVE} ] &&
+       [   -d ${SRC_ARCHIVE} ] &&
+       [   -f ${SRC_ARCHIVE}/$FILE ]; then
+      cp ${SRC_ARCHIVE}/$FILE .
+      echo "$FILE: -- copied from $SRC_ARCHIVE"
+      fromARCHIVE=1
+    else
+      echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}"
+      fromARCHIVE=0
+      # If the file does not exist in /sources download a fresh one
+      if [ ! -f $FILE ] ; then
+        if ! wget $URL1 && ! wget $URL2 ; then
+          gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING" 
+          continue 
+        fi
+      fi
+    fi
+
+      # Is there a MD5SUM to validate the file against.
+    if [[ "$HAVEMD5" = "1" ]] ; then
+        # IF the md5sum does not match the existing files
+      if ! echo "$MD5" | md5sum -c - >/dev/null ; then
+        [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
+        [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
+          # Remove the old file and download a new one
+        rm -fv $FILE
+          # Force generation of MD5SUM and storage in SRC_ARCHIVE 
+        fromARCHIVE=0; HAVEMD5=0
+          # Try and retrieve the file
+          # I swap server priority to allow for corruption on the primary site
+        if ! wget -N $URL1 && ! wget -N $URL2 ; then
+          gs_wrt_message "$FILE not found on the servers.. SKIPPING"
+          continue
+        fi
+      fi
+    fi
+
+      # Validate the MD5SUM one last time
+    if [[ "$HAVEMD5" = "1" ]] && ! echo "$MD5" | md5sum -c - >/dev/null ; then
+      gs_wrt_message "$FILE does not match MD5SUMS value"
+    fi
+
+      # Generate a fresh MD5SUM for this file
+    if [[ "$HAVEMD5" = "0" ]] ; then
+      echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
+      echo "NEW MD5SUM $(md5sum $FILE)" >> MISSING_FILES.DMP    
+    fi
+
+      # Good or bad we write the original md5sum to a file
+    echo "$MD5" >> MD5SUMS-$VERSION
+
+     # Copy the freshly downloaded file
+     # to the source archive.
+   if [ ! -z ${SRC_ARCHIVE}  ] &&
+      [   -d ${SRC_ARCHIVE}  ] &&
+      [   -w ${SRC_ARCHIVE}  ] &&
+      [ "$fromARCHIVE" = "0" ] ; then
+        echo "Storing file:<$FILE> in the package archive"
+        cp -f $FILE ${SRC_ARCHIVE}
+   fi
+
+  done
+
+  if [[ -s MISSING_FILES.DMP ]]; then
+    echo  -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
+      # Do not allow the automatic exection of the Makefile.
+    echo     "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
+    RUNMAKE=0
+  fi
+}
+
+
+
+#----------------------------#
+get_sourcesX() {              # Download file, write name to MISSING_FILES.DMP if an error
 #----------------------------#
   local saveIFS=$IFS