source: common/libs/func_download_pkgs@ c785566

new_features
Last change on this file since c785566 was 7bd0866, checked in by Pierre Labastie <pierre@…>, 8 years ago

Merge trunk up to revision 3864

  • Property mode set to 100644
File size: 6.5 KB
Line 
1#!/bin/bash
2
3# $Id$
4
5
6#----------------------------#
7get_sources() { # Download file, write name to MISSING_FILES.DMP if an error
8#----------------------------#
9 local saveIFS=$IFS
10 local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE WGETPARAM
11
12 WGETPARAM=""
13 if [[ "${RETRYSRCDOWNLOAD}" = "y" ]] ; then
14 WGETPARAM+="--retry-connrefused"
15 fi
16 WGETPARAM+=" --tries ${RETRYDOWNLOADCNT}"
17 WGETPARAM+=" --timeout ${DOWNLOADTIMEOUT}"
18
19 # Test if the packages must be downloaded
20 [ ! "$GETPKG" = "y" ] && return
21
22 gs_wrt_message(){
23 echo "${RED}$1${OFF}"
24 echo "$1" >> MISSING_FILES.DMP
25 }
26 # Housekeeping
27 [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
28 cd $BUILDDIR/sources
29 # If using CLFS, /sources is writable by all, but with sticky bit,
30 # and user does not hold MD5SUMS nor the other files, so use sudo
31 [[ -f MD5SUMS ]] && sudo rm MD5SUMS
32 [[ -f MISSING_FILES.DMP ]] && sudo rm MISSING_FILES.DMP
33 [[ -f urls.lst ]] && sudo rm urls.lst
34
35 # Generate URLs file
36 create_urls
37
38 IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
39 for line in `cat urls.lst`; do
40 IFS=$saveIFS # Restore the system defaults
41
42 # Skip some packages if they aren't needed
43 case $line in
44 */tree* | */gcc-testsuite* ) #unneeded now?
45 [[ "$TEST" = "0" ]] && continue
46 ;;
47 */vim-*-lang* )
48 [[ "$VIMLANG" = "0" ]] && continue
49 ;;
50 esac
51
52 # Locations
53 URL1=`echo $line | cut -d" " -f2` # Preferred URL
54 URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL
55 FILE=`basename $URL1` # File name
56 BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value
57
58 # Validation pair
59 MD5="$BOOKMD5 $FILE"
60 HAVEMD5=1
61
62 set -e
63 # If the file exists in the archive copy it to the
64 # $BUILDDIR/sources dir. MD5SUM will be validated later.
65 if [ ! -z ${SRC_ARCHIVE} ] &&
66 [ -d ${SRC_ARCHIVE} ] &&
67 [ -f ${SRC_ARCHIVE}/$FILE ]; then
68 cp ${SRC_ARCHIVE}/$FILE .
69 echo "$FILE: -- copied from $SRC_ARCHIVE"
70 fromARCHIVE=1
71 else
72 fromARCHIVE=0
73 # If the file does not exist yet in /sources download a fresh one
74 if [ ! -f $FILE ] ; then
75 if [[ ! ("$SRC_ARCHIVE" = "") ]] ; then
76 echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE} or ${BUILDDIR}/sources${OFF}"
77 else
78 echo "${BOLD}${YELLOW}$FILE: not found in ${BUILDDIR}/sources${OFF}"
79 fi
80 if ! wget $URL1 $WGETPARAM && ! wget $URL2 $WGETPARAM ; then
81 gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
82 continue
83 fi
84 else
85 echo "${BOLD}${YELLOW}$FILE: using cached file in ${BUILDDIR}/sources${OFF}"
86 fi
87 fi
88
89 # Deal with udev and bootscripts m5sum issue
90 [[ $BOOKMD5 = "BOOTSCRIPTS-MD5SUM" ]] && continue
91 [[ $BOOKMD5 = "UDEV-MD5SUM" ]] && continue
92 [[ $BOOKMD5 = "LFS-NETSCRIPTS-MD5SUM" ]] && continue
93
94 # IF the md5sum does not match the existing files
95 if ! echo "$MD5" | md5sum -c - >/dev/null ; then
96 [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
97 [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
98 # Remove the old file and download a new one
99 rm -fv $FILE
100 # Force storage in SRC_ARCHIVE
101 fromARCHIVE=0;
102 # Try to retrieve again the file. Servers in reverse order.
103 if ! wget $URL2 $WGETPARAM && ! wget $URL1 $WGETPARAM ; then
104 gs_wrt_message "$FILE not found on the servers.. SKIPPING"
105 continue
106 fi
107 fi
108
109 # Validate the MD5SUM one last time
110 if ! echo "$MD5" | md5sum -c - >/dev/null ; then
111 gs_wrt_message "$FILE does not match MD5SUMS value"
112 # Force generation of MD5SUM
113 HAVEMD5=0
114 fi
115
116 # Generate a fresh MD5SUM for this file
117 if [[ "$HAVEMD5" = "0" ]] ; then
118 echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
119 echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP
120 fi
121
122 # Good or bad we write the original md5sum to a file
123 echo "$MD5" >> MD5SUMS
124
125 # Copy the freshly downloaded file
126 # to the source archive.
127 if [ ! -z ${SRC_ARCHIVE} ] &&
128 [ -d ${SRC_ARCHIVE} ] &&
129 [ -w ${SRC_ARCHIVE} ] &&
130 [ "$fromARCHIVE" = "0" ] ; then
131 echo "Storing file:<$FILE> in the package archive"
132 cp -f $FILE ${SRC_ARCHIVE}
133 fi
134
135 done
136
137 if [[ -s MISSING_FILES.DMP ]]; then
138 echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
139 # Do not allow the automatic execution of the Makefile.
140 echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
141 RUNMAKE="n"
142 fi
143}
144
145#----------------------------#
146create_urls() { #
147#----------------------------#
148 cd $JHALFSDIR
149
150 case ${PROGNAME} in
151 clfs*)
152 echo -n "Creating CLFS <${ARCH}> specific URLs file"
153 xsltproc --nonet --xinclude \
154 --stringparam server $SERVER \
155 --stringparam family clfs \
156 -o $BUILDDIR/sources/urls.lst \
157 urls.xsl \
158 $BOOK/BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
159 echo " ...OK"
160 ;;
161 hlfs)
162 echo -n "Creating HLFS <${MODEL}> + <${KERNEL}> specific URLs file"
163 xsltproc --nonet --xinclude \
164 --stringparam server $SERVER \
165 --stringparam family lfs \
166 --stringparam model $MODEL \
167 --stringparam kernel $KERNEL \
168 -o $BUILDDIR/sources/urls.lst \
169 urls.xsl \
170 $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1
171 echo " ...OK"
172 ;;
173 lfs)
174 echo -n "Creating LFS specific URLs file"
175 xsltproc --nonet --xinclude \
176 --stringparam server $SERVER \
177 --stringparam family lfs \
178 --stringparam pkgmngt $PKGMNGT \
179 --stringparam revision $INITSYS \
180 --output ../sources/urls.lst \
181 urls.xsl \
182 $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1
183 echo " ...OK"
184 ;;
185 esac
186
187 cd $BUILDDIR/sources
188
189 if [[ "${CUSTOM_TOOLS}" = "y" ]]; then
190 add_CustomToolsURLS
191 fi
192
193}
194
Note: See TracBrowser for help on using the repository browser.