source: common/libs/func_download_pkgs@ a690d42

ablfs-more legacy trunk
Last change on this file since a690d42 was fd4a798, checked in by Pierre Labastie <pierre.labastie@…>, 3 years ago

Remove $Id$ comments, they are useless with git

  • Property mode set to 100644
File size: 6.3 KB
RevLine 
[fe30c61]1#!/bin/bash
2
3#----------------------------#
4get_sources() { # Download file, write name to MISSING_FILES.DMP if an error
5#----------------------------#
6 local saveIFS=$IFS
[de63126]7 local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE WGETPARAM
8
9 WGETPARAM=""
10 if [[ "${RETRYSRCDOWNLOAD}" = "y" ]] ; then
11 WGETPARAM+="--retry-connrefused"
12 fi
13 WGETPARAM+=" --tries ${RETRYDOWNLOADCNT}"
14 WGETPARAM+=" --timeout ${DOWNLOADTIMEOUT}"
[fe30c61]15
16 # Test if the packages must be downloaded
17 [ ! "$GETPKG" = "y" ] && return
18
19 gs_wrt_message(){
20 echo "${RED}$1${OFF}"
21 echo "$1" >> MISSING_FILES.DMP
22 }
23 # Housekeeping
24 [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
25 cd $BUILDDIR/sources
[bb4b5c3]26 # If using CLFS, /sources is writable by all, but with sticky bit,
27 # and user does not hold MD5SUMS nor the other files, so use sudo
28 [[ -f MD5SUMS ]] && sudo rm MD5SUMS
29 [[ -f MISSING_FILES.DMP ]] && sudo rm MISSING_FILES.DMP
30 [[ -f urls.lst ]] && sudo rm urls.lst
[fe30c61]31
32 # Generate URLs file
33 create_urls
34
35 IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
36 for line in `cat urls.lst`; do
37 IFS=$saveIFS # Restore the system defaults
38
39 # Locations
40 URL1=`echo $line | cut -d" " -f2` # Preferred URL
41 URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL
42 FILE=`basename $URL1` # File name
43 BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value
44
45 # Validation pair
46 MD5="$BOOKMD5 $FILE"
47 HAVEMD5=1
48
49 set -e
50 # If the file exists in the archive copy it to the
51 # $BUILDDIR/sources dir. MD5SUM will be validated later.
52 if [ ! -z ${SRC_ARCHIVE} ] &&
53 [ -d ${SRC_ARCHIVE} ] &&
54 [ -f ${SRC_ARCHIVE}/$FILE ]; then
55 cp ${SRC_ARCHIVE}/$FILE .
56 echo "$FILE: -- copied from $SRC_ARCHIVE"
57 fromARCHIVE=1
58 else
59 fromARCHIVE=0
60 # If the file does not exist yet in /sources download a fresh one
61 if [ ! -f $FILE ] ; then
[9d55d85]62 if [[ ! ("$SRC_ARCHIVE" = "") ]] ; then
[7072e1f]63 echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE} or ${BUILDDIR}/sources${OFF}"
[9d55d85]64 else
[7072e1f]65 echo "${BOLD}${YELLOW}$FILE: not found in ${BUILDDIR}/sources${OFF}"
[9d55d85]66 fi
[de63126]67 if ! wget $URL1 $WGETPARAM && ! wget $URL2 $WGETPARAM ; then
[fe30c61]68 gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
69 continue
70 fi
[7072e1f]71 else
72 echo "${BOLD}${YELLOW}$FILE: using cached file in ${BUILDDIR}/sources${OFF}"
[fe30c61]73 fi
74 fi
75
[5210ae1]76 # Deal with udev and bootscripts m5sum issue
[0700131]77 [[ $BOOKMD5 = "BOOTSCRIPTS-MD5SUM" ]] && continue
78 [[ $BOOKMD5 = "UDEV-MD5SUM" ]] && continue
79 [[ $BOOKMD5 = "LFS-NETSCRIPTS-MD5SUM" ]] && continue
[2718053]80 [[ $BOOKMD5 = "CUSTOM-PATCH-MD5SUM" ]] && continue
[5210ae1]81
[fe30c61]82 # IF the md5sum does not match the existing files
83 if ! echo "$MD5" | md5sum -c - >/dev/null ; then
84 [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
85 [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
86 # Remove the old file and download a new one
87 rm -fv $FILE
88 # Force storage in SRC_ARCHIVE
89 fromARCHIVE=0;
90 # Try to retrieve again the file. Servers in reverse order.
[de63126]91 if ! wget $URL2 $WGETPARAM && ! wget $URL1 $WGETPARAM ; then
[fe30c61]92 gs_wrt_message "$FILE not found on the servers.. SKIPPING"
93 continue
94 fi
95 fi
96
97 # Validate the MD5SUM one last time
98 if ! echo "$MD5" | md5sum -c - >/dev/null ; then
99 gs_wrt_message "$FILE does not match MD5SUMS value"
100 # Force generation of MD5SUM
101 HAVEMD5=0
102 fi
103
104 # Generate a fresh MD5SUM for this file
105 if [[ "$HAVEMD5" = "0" ]] ; then
106 echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
107 echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP
108 fi
109
110 # Good or bad we write the original md5sum to a file
[93fd2d0]111 echo "$MD5" >> MD5SUMS
[fe30c61]112
113 # Copy the freshly downloaded file
114 # to the source archive.
115 if [ ! -z ${SRC_ARCHIVE} ] &&
116 [ -d ${SRC_ARCHIVE} ] &&
117 [ -w ${SRC_ARCHIVE} ] &&
118 [ "$fromARCHIVE" = "0" ] ; then
119 echo "Storing file:<$FILE> in the package archive"
120 cp -f $FILE ${SRC_ARCHIVE}
121 fi
122
123 done
124
125 if [[ -s MISSING_FILES.DMP ]]; then
126 echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
127 # Do not allow the automatic execution of the Makefile.
128 echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
129 RUNMAKE="n"
130 fi
131}
132
133#----------------------------#
134create_urls() { #
135#----------------------------#
136 cd $JHALFSDIR
137
138 case ${PROGNAME} in
[809a821]139 clfs*)
[fe30c61]140 echo -n "Creating CLFS <${ARCH}> specific URLs file"
[a4acb12]141 xsltproc --nonet --xinclude \
142 --stringparam server "$SERVER" \
143 --stringparam family clfs \
144 -o $BUILDDIR/sources/urls.lst \
145 urls.xsl \
[809a821]146 $BOOK/BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
[fe30c61]147 echo " ...OK"
148 ;;
149 hlfs)
[139c34e]150 echo -n "Creating HLFS <${MODEL}> + <${KERNEL}> specific URLs file"
[a4acb12]151 xsltproc --nonet --xinclude \
152 --stringparam server "$SERVER" \
153 --stringparam family lfs \
154 --stringparam model "$MODEL" \
155 --stringparam kernel "$KERNEL" \
156 -o $BUILDDIR/sources/urls.lst \
157 urls.xsl \
[fe30c61]158 $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1
159 echo " ...OK"
160 ;;
161 lfs)
162 echo -n "Creating LFS specific URLs file"
[a4acb12]163 xsltproc --nonet --xinclude \
164 --stringparam server "$SERVER" \
165 --stringparam family lfs \
166 --stringparam pkgmngt "$PKGMNGT" \
167 --stringparam revision "$INITSYS" \
168 --output ../sources/urls.lst \
169 urls.xsl \
[fe30c61]170 $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1
171 echo " ...OK"
172 ;;
173 esac
174
175 cd $BUILDDIR/sources
176
177 if [[ "${CUSTOM_TOOLS}" = "y" ]]; then
178 add_CustomToolsURLS
179 fi
180
181}
182
Note: See TracBrowser for help on using the repository browser.