source: common/common-functions@ 93135fd

1.0 2.3 2.3.x 2.4 ablfs ablfs-more legacy new_features trunk
Last change on this file since 93135fd was 93135fd, checked in by George Boudreau <georgeb@…>, 18 years ago

Rewrote the package download function get_sources

  • Property mode set to 100644
File size: 26.7 KB
Line 
1#!/bin/bash
2
3# $Id$
4
5set +e
6
7# VT100 colors
8declare -r BLACK=$'\e[1;30m'
9declare -r DK_GRAY=$'\e[0;30m'
10
11declare -r RED=$'\e[31m'
12declare -r GREEN=$'\e[32m'
13declare -r YELLOW=$'\e[33m'
14declare -r BLUE=$'\e[34m'
15declare -r MAGENTA=$'\e[35m'
16declare -r CYAN=$'\e[36m'
17declare -r WHITE=$'\e[37m'
18
19declare -r OFF=$'\e[0m'
20declare -r BOLD=$'\e[1m'
21declare -r REVERSE=$'\e[7m'
22declare -r HIDDEN=$'\e[8m'
23
24declare -r tab_=$'\t'
25declare -r nl_=$'\n'
26
27declare -r DD_BORDER="${BOLD}==============================================================================${OFF}"
28declare -r SD_BORDER="${BOLD}------------------------------------------------------------------------------${OFF}"
29declare -r STAR_BORDER="${BOLD}******************************************************************************${OFF}"
30
31# bold yellow > < pair
32declare -r R_arrow=$'\e[1;33m>\e[0m'
33declare -r L_arrow=$'\e[1;33m<\e[0m'
34
35
36usage() {
37cat <<- -EOF-
38${DD_BORDER}
39${BOLD}
40 Usage: $0 ${BOLD}[OPTION]
41
42Options:
43${BOLD} -h, --help${OFF}
44 print this help, then exit
45
46${BOLD} -V, --version${OFF}
47 print version information, then exit
48
49${BOLD} -B, --book VER${OFF}
50 checkout VER version of the book. Supported versions at this time are:
51 dev* | trunk | SVN aliases for Development {C,H,B}LFS
52
53${BOLD} -D --directory DIR${OFF}
54 use DIR directory for building ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF}; all files jhalfs produces
55 will be in the directory DIR/jhalfs.
56
57${BOLD} -G, --get-packages${OFF}
58 download the packages and patches. This assumes that the server declared
59 in the configuration file has the proper packages and patches for the
60 book version being processed.
61
62${BOLD} -T, --testsuites N ${OFF}
63 Run test suites [0-3]
64 0 = none
65 1 = only chapter06 Glibc, GCC and Binutils testsuites
66 2 = all chapter06 testsuites (in BLFS, alias to 1)
67 3 = all chapter05 and chapter06 testsuites
68 In BLFS, 2 and 3 are aliases to 1
69 In CLFS, 3 is an alias to 2
70
71${BOLD} -W, --working-copy DIR${OFF}
72 use the local working copy placed in DIR as the $(echo $PROGNAME | tr [a-z] [A-Z]) book
73-EOF-
74
75[[ ${PROGNAME} != "blfs" ]] &&
76cat <<- -EOF-
77
78${BOLD} -C, --comparasion TYPE${OFF}
79 do iterative comparison analysis. That will take several time due that
80 the final system packages wil be rebuilded the times defined in the
81 ITERATIONS config option.
82 Analisys types allowed are:
83 ICA = do the ICA analysis designed by Greg Schafer
84 farce = do the farce analysis designed by Ken Moffat
85 both = perfom both ICA and farce analysis
86
87${BOLD} -F, --fstab FILE${OFF}
88 use FILE as the /etc/fstab file for the ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF} system. If not specified,
89 a default /etc/fstab file with dummy values is created.
90
91${BOLD} -K, --kernel-config FILE${OFF}
92 use the kernel configuration file specified in FILE to build the kernel.
93 if the file is not found, or if not specified, the kernel build is skipped.
94
95${BOLD} -M, --run-make${OFF}
96 run make on the generated Makefile
97
98${BOLD} -R --rebuild${OFF}
99 clean the build directory before to perfom any other task. The directory
100 is cleaned only if it was populated by a previous jhalfs run.
101-EOF-
102
103[[ ${PROGNAME} = "clfs" ]] &&
104cat <<- -EOF-
105
106${BOLD} -A, --arch ARCH ${OFF}
107 Select the TARGET architecture, valid selections are:
108 32bit builds
109 x86, i486, i586, ppc, mips, mipsel, sparc
110 64bit builds
111 x86_64-64, mips64-64, mipsel64-64, sparc64-64, alpha
112 64bit multi-lib
113 x86_64, mips64, mipsel64, sparc64, ppc64
114
115${BOLD} --boot-config FILE ${OFF}
116 The configuration file for the bootstrap kernel if method=boot
117
118${BOLD} --method BUILDMETHOD ${OFF}
119 Select the build method, chroot or boot
120-EOF-
121
122[[ ${PROGNAME} = "hlfs" ]] &&
123cat <<- -EOF-
124
125${BOLD} --model STYLE ${OFF}
126 Select the library model for the HLFS system
127 Valid choices are: glibc or uclibc
128-EOF-
129
130[[ ${PROGNAME} = "blfs" ]] &&
131cat <<- -EOF-
132
133${BOLD} --dependencies 0/1/2${OFF}
134 add dependencies of type TYPE to the build tree.
135 If not set, both required and recommended are used.
136
137 Possible values are:
138 0 - required only required dependecies are used
139 1 - recommended both required a recommended dependencies are used
140 2 - optional all dependencies are used
141-EOF-
142
143cat <<- -EOF-
144${DD_BORDER}
145-EOF-
146 exit
147}
148
149version="
150${BOLD} \"jhalfs\"${OFF} builder tool (development) \$Rev$
151\$Date$
152
153${BOLD} \"${PROGNAME}\"${OFF} script module
154
155Written by George Boudreau,
156 Manuel Canales Esparcia,
157 Jeremy Huntwork
158
159This program is published under the ${BOLD}Gnu General Public License, Version 2.${OFF}
160"
161
162
163no_empty_builddir() {
164 'clear'
165cat <<- -EOF-
166${DD_BORDER}
167
168${tab_}${tab_}${BOLD}${RED}W A R N I N G${OFF}
169 Looks like the \$BUILDDIR directory contains subdirectories
170 from a previous HLFS build.
171
172 Please format the partition mounted on \$BUILDDIR or set
173 a different build directory before running jhalfs.
174${OFF}
175${DD_BORDER}
176-EOF-
177 exit
178}
179
180
181help="${nl_}Try '$0 --help' for more information."
182
183
184exit_missing_arg="\
185echo \"Option '\$1' requires an argument\" >&2
186echo \"\$help\" >&2
187exit 1"
188
189no_dl_client="\
190echo \"Could not find a way to download the ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF} sources.\" >&2
191echo \"Attempting to continue.\" >&2"
192
193HEADER="# This file is automatically generated by jhalfs
194# DO NOT EDIT THIS FILE MANUALLY
195#
196# Generated on `date \"+%F %X %Z\"`"
197
198
199
200
201
202#----------------------------------#
203wrt_target() { # Create target and initialize log file
204#----------------------------------#
205 local i=$1
206 local PREV=$2
207(
208cat << EOF
209
210$i: $PREV
211 @\$(call echo_message, Building)
212 @echo -e "\n\`date\`\n\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >logs/$this_script
213EOF
214) >> $MKFILE.tmp
215}
216
217
218#----------------------------------#
219wrt_unpack() { # Unpack and set 'ROOT' var
220#----------------------------------#
221 local FILE=$1
222(
223cat << EOF
224 @\$(call unpack,$FILE)
225 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
226 echo "export PKGDIR=\$(MOUNT_PT)\$(SRC)/\$\$ROOT" > envars && \\
227 chown -R lfs \$(MOUNT_PT)\$(SRC)/\$\$ROOT
228EOF
229) >> $MKFILE.tmp
230}
231
232
233#----------------------------------#
234wrt_unpack2() { #
235#----------------------------------#
236 local FILE=$1
237(
238cat << EOF
239 @\$(call unpack2,$FILE)
240 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
241 echo "export PKGDIR=\$(SRC)/\$\$ROOT" > envars
242EOF
243) >> $MKFILE.tmp
244}
245
246
247#=============================#
248wrt_unpack3() { #
249#=============================#
250 local FILE=$1
251(
252cat << EOF
253 @\$(call unpack3,$FILE)
254 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
255 echo "export PKGDIR=\$(SRC)/\$\$ROOT" > envars
256EOF
257) >> $MKFILE.tmp
258}
259
260
261
262#----------------------------------#
263wrt_target_vars() { # Target vars for hlfs (cross-build method)
264#----------------------------------#
265(
266cat << EOF
267 @echo "export target=$(uname -m)-${TARGET}" >> envars && \\
268 echo "export ldso=/lib/${LOADER}" >> envars
269EOF
270) >> $MKFILE.tmp
271
272}
273
274
275#----------------------------------#
276wrt_run_as_su() { # Execute script inside time { }, footer to log file
277#----------------------------------#
278 local this_script=$1
279 local file=$2
280(
281cat << EOF
282 @( time { su - lfs -c "source /home/lfs/.bashrc && $JHALFSDIR/${PROGNAME}-commands/$file" >>logs/$this_script 2>&1 ; } ) 2>>logs/$this_script && \\
283 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/$this_script
284EOF
285) >> $MKFILE.tmp
286}
287
288
289#----------------------------------#
290wrt_run_as_root() { # Some scripts must be run as root..
291#----------------------------------#
292 local this_script=$1
293 local file=$2
294(
295cat << EOF
296 @( time { export LFS=\$(MOUNT_PT) && ${PROGNAME}-commands/$file >>logs/$this_script 2>&1 ; } ) 2>>logs/$this_script && \\
297 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/$this_script
298EOF
299) >> $MKFILE.tmp
300}
301
302
303#=============================#
304wrt_run_as_root2() { #
305#=============================#
306 local this_script=$1
307 local file=$2
308(
309cat << EOF
310 @( time { source envars && ${PROGNAME}-commands/$file >>logs/$this_script 2>&1 ; } ) 2>>logs/$this_script && \\
311 echo -e "\nKB: \`du -skx --exclude=jhalfs \`\n" >>logs/$this_script
312EOF
313) >> $MKFILE.tmp
314}
315
316
317
318#----------------------------------#
319wrt_run_as_chroot1() { #
320#----------------------------------#
321 local this_script=$1
322 local file=$2
323(
324 cat << EOF
325 @( time { \$(CHROOT1) 'cd /jhalfs && source envars && /jhalfs/${PROGNAME}-commands/$file >>/jhalfs/logs/${this_script} 2>&1' ; } ) 2>>logs/$this_script && \\
326 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/${this_script}
327EOF
328) >> $MKFILE.tmp
329}
330
331
332#----------------------------------#
333wrt_run_as_chroot2() { #
334#----------------------------------#
335 local this_script=$1
336 local file=$2
337(
338cat << EOF
339 @( time { \$(CHROOT2) 'cd /jhalfs && source envars && /jhalfs/${PROGNAME}-commands/$file >>/jhalfs/logs/${this_script} 2>&1' ; } ) 2>>logs/$this_script && \\
340 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/${this_script}
341EOF
342) >> $MKFILE.tmp
343}
344
345
346#----------------------------------#
347wrt_copy_fstab() { #
348#----------------------------------#
349 local i=$1
350(
351 cat << EOF
352 @cp -v $FSTAB \$(MOUNT_PT)/etc/fstab >>logs/$i 2>&1
353EOF
354) >> $MKFILE.tmp
355}
356
357#----------------------------------#
358wrt_copy_fstab2() { #
359#----------------------------------#
360 local i=$1
361(
362 cat << EOF
363 @cp -v /sources/fstab /etc/fstab >>logs/$i 2>&1
364EOF
365) >> $MKFILE.tmp
366}
367
368
369#----------------------------------#
370wrt_remove_build_dirs() { #
371#----------------------------------#
372 local name=$1
373(
374cat << EOF
375 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
376 rm -r \$(MOUNT_PT)\$(SRC)/\$\$ROOT && \\
377 if [ -e \$(MOUNT_PT)\$(SRC)/$name-build ]; then \\
378 rm -r \$(MOUNT_PT)\$(SRC)/$name-build; \\
379 fi;
380EOF
381) >> $MKFILE.tmp
382}
383
384
385#----------------------------------#
386wrt_remove_build_dirs2() { #
387#----------------------------------#
388 local name=$1
389(
390cat << EOF
391 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
392 rm -r \$(SRC)/\$\$ROOT && \\
393 if [ -e \$(SRC)/$name-build ]; then \\
394 rm -r \$(SRC)/$name-build; \\
395 fi;
396EOF
397) >> $MKFILE.tmp
398}
399
400
401#----------------------------------#
402wrt_report() { #
403#----------------------------------#
404(
405cat << EOF
406
407create-sbu_du-report: $PREV
408 @\$(call echo_message, Building)
409 @./create-sbu_du-report.sh logs $VERSION
410 @\$(call echo_report,$VERSION-SBU_DU-$(date --iso-8601).report)
411 @touch \$@
412EOF
413) >> $MKFILE.tmp
414
415chapter789="$chapter789 create-sbu_du-report"
416}
417
418
419#----------------------------#
420run_make() {
421#----------------------------#
422 # Test if make must be run.
423 if [ "$RUNMAKE" = "1" ] ; then
424 # Test to make sure we're running the build as root
425 if [ "$UID" != "0" ] ; then
426 echo "You must be logged in as root to successfully build the system."
427 exit 1
428 fi
429 # Build the system
430 if [ -e $MKFILE ] ; then
431 echo -ne "Building the system...\n"
432 cd $JHALFSDIR && make
433 echo -ne "done\n"
434 fi
435 fi
436}
437
438
439#----------------------------#
440clean_builddir() {
441#----------------------------#
442 # Test if the clean must be done.
443 if [ "$CLEAN" = "1" ] ; then
444 # Test to make sure we're running the clean as root
445 if [ "$UID" != "0" ] ; then
446 echo "You must be logged in as root to clean the build directory."
447 exit 1
448 fi
449 # Test to make sure that the build directory was populated by jhalfs
450 if [ ! -d $JHALFSDIR ] || [ ! -d $BUILDDIR/sources ] ; then
451 echo "Looks like $BUILDDIR was not populated by a previous jhalfs run."
452 exit 1
453 else
454 # Clean the build directory
455 echo -ne "Cleaning $BUILDDIR...\n"
456 rm -rf $BUILDDIR/{bin,boot,dev,etc,home,lib,media,mnt,opt,proc,root,sbin,srv,sys,tmp,tools,cross-tools,usr,var}
457 echo -ne "Cleaning $JHALFSDIR...\n"
458 rm -rf $JHALFSDIR/{0*,1*,envars,sources-dir,*commands,logs,Makefile,*.xsl,makefile-functions,packages,patches}
459 echo -ne "Cleaning remainig extracted sources in $BUILDDIR/sources...\n"
460 rm -rf `find $BUILDDIR/sources/* -maxdepth 0 -type d`
461 echo -ne "done\n"
462 fi
463 fi
464}
465
466#----------------------------#
467get_book() {
468#----------------------------#
469 cd $JHALFSDIR
470
471 if [ -z $WC ] ; then
472 # Check for Subversion instead of just letting the script hit 'svn' and fail.
473 test `type -p svn` || eval "echo \"This feature requires Subversion.\"
474 exit 1"
475 echo -n "Downloading the $PROGNAME document, $LFSVRS version... "
476
477 case $PROGNAME in
478 lfs) svn_root="LFS" ;;
479 hlfs) svn_root="HLFS" ;;
480 clfs) svn_root="cross-lfs" ;;
481 blfs) svn_root="BLFS" ;;
482 *) echo "BOOK not defined in function <get_book>"
483 exit 1 ;;
484 esac
485 # Grab a fresh book if it's missing, otherwise, update it from the
486 # repo. If we've already extracted the commands, move on to getting the
487 # sources.
488 if [ -d ${PROGNAME}-$LFSVRS ] ; then
489 cd ${PROGNAME}-$LFSVRS
490 if LC_ALL=C svn up | grep -q At && test -d $JHALFSDIR/${PROGNAME}-commands && \
491 test -f $JHALFSDIR/packages ; then
492 echo -ne "done\n"
493 # Set the canonical book version
494 cd $JHALFSDIR
495 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
496 get_sources
497 else
498 echo -ne "done\n"
499 # Set the canonical book version
500 cd $JHALFSDIR
501 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
502 extract_commands
503 fi
504 else
505 case $LFSVRS in
506 development)
507 svn co $SVN/${svn_root}/trunk/BOOK ${PROGNAME}-$LFSVRS >>$LOGDIR/$LOG 2>&1 ;;
508 esac
509 echo -ne "done\n"
510 # Set the canonical book version
511 cd $JHALFSDIR
512 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
513 extract_commands
514 fi
515 else
516 echo -ne "Using $BOOK as book's sources ...\n"
517 # Set the canonical book version
518 cd $JHALFSDIR
519 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
520 extract_commands
521 fi
522}
523
524#----------------------------#
525extract_commands() { #
526#----------------------------#
527
528 # Check for libxslt instead of just letting the script hit 'xsltproc' and fail.
529 test `type -p xsltproc` || eval "echo \"This feature requires libxslt.\"
530 exit 1"
531
532 cd $JHALFSDIR
533 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
534
535 # Start clean
536 if [ -d ${PROGNAME}-commands ]; then
537 rm -rf ${PROGNAME}-commands
538 mkdir -v ${PROGNAME}-commands
539 fi
540 echo -n "Extracting commands for"
541
542 # Dump the commands in shell script form from the HLFS book.
543 case ${PROGNAME} in
544 clfs)
545 echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture"
546 xsltproc --nonet \
547 --xinclude \
548 --stringparam method $METHOD \
549 --stringparam testsuite $TEST \
550 --stringparam vim-lang $VIMLANG \
551 --stringparam timezone $TIMEZONE \
552 --stringparam page $PAGE \
553 --stringparam lang $LANG \
554 --stringparam keymap $KEYMAP \
555 -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1
556 ;;
557 hlfs)
558 echo -n " ${L_arrow}${BOLD}$MODEL${R_arrow} HLFS libc implementation"
559 xsltproc --nonet \
560 --xinclude \
561 --stringparam model $MODEL \
562 --stringparam testsuite $TEST \
563 --stringparam timezone $TIMEZONE \
564 --stringparam page $PAGE \
565 --stringparam lang $LANG \
566 --stringparam lc_all $LC_ALL \
567 --stringparam keymap $KEYMAP \
568 --stringparam grsecurity_host $GRSECURITY_HOST \
569 -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1
570 ;;
571 lfs)
572 echo -n " ${L_arrow}${BOLD}LFS${R_arrow} build"
573 xsltproc --nonet \
574 --xinclude \
575 --stringparam testsuite $TEST \
576 --stringparam vim-lang $VIMLANG \
577 --stringparam timezone $TIMEZONE \
578 --stringparam page $PAGE \
579 --stringparam lang $LANG \
580 -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1
581 ;;
582 blfs)
583 echo -n " ${L_arrow}${BOLD}BLFS${R_arrow} build"
584 xsltproc --nonet \
585 --xinclude \
586 --stringparam testsuite $TEST \
587 --stringparam server $SERVER \
588 -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1
589 ;;
590 *) exit 1 ;;
591 esac
592
593 echo " ...OK"
594
595 # Make the scripts executable.
596 chmod -R +x $JHALFSDIR/${PROGNAME}-commands
597
598 # Create the packages file. We need it for proper Makefile creation
599 rm -f packages
600 case "${PROGNAME}" in
601 clfs)
602 echo -n "Creating <${PROGNAME}> specific packages file"
603 grep "\-version " $BOOK/packages.ent | sed -e 's@<!ENTITY @@' \
604 -e 's@">@"@' \
605 -e '/generic/d' > packages
606 echo " ...OK"
607 ;;
608
609 hlfs)
610 echo -n "Creating <${PROGNAME}> specific packages file"
611 grep "\-version" $BOOK/general.ent | sed -e 's@<!ENTITY @@' \
612 -e 's@">@"@' \
613 -e '/generic/d' >> packages
614 echo " ...OK"
615 ;;
616
617 lfs)
618 echo -n "Creating <${PROGNAME}> specific packages file"
619 grep "\-version" $BOOK/general.ent | sed -e 's@<!ENTITY @@' \
620 -e 's@">@"@' \
621 -e '/generic/d' >> packages
622 echo " ...OK"
623 ;;
624 esac
625
626 # Done. Moving on...
627 get_sources
628}
629
630#----------------------------#
631get_sources() { # Download file, write name to MISSING_FILES.DMP if an error
632#----------------------------#
633 local saveIFS=$IFS
634 local IFS line URL1 URL2 FILE MD5 HAVEMD5 fromARCHIVE
635
636 # Test if the packages must be downloaded
637 [ ! "$GETPKG" = "1" ] && return
638
639 gs_wrt_message(){
640 echo "${RED}$1${OFF}"
641 echo "$1" >> MISSING_FILES.DMP
642 }
643 # Housekeeping
644 [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
645 cd $BUILDDIR/sources
646 [[ -f MD5SUMS ]] && rm MD5SUMS
647 [[ -f MD5SUMS-$VERSION ]] && rm MD5SUMS-$VERSION
648 [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP
649 [[ -f urls.lst ]] && rm urls.lst
650
651 # Download a fresh MD5SUMS file
652 wget $SERVER/pub/lfs/conglomeration/MD5SUMS
653
654 # Generate URLs file
655 create_urls
656
657 IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
658 for line in `cat urls.lst`; do
659 IFS=$saveIFS # Restore the system defaults
660
661 # Skip some packages if they aren't needed
662 case $line in
663 */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* )
664 [[ "$TEST" = "0" ]] && continue
665 ;;
666 */vim-*-lang* )
667 [[ "$VIMLANG" = "0" ]] && continue
668 ;;
669 esac
670
671 # Locations
672 URL1=`echo $line | cut -d" " -f1` # Upstream URL
673 URL2=`echo $line | cut -d" " -f2` # Fallback URL
674 FILE=`basename $URL2` # File name
675
676 # Find the md5 sum for this package.
677 set +e
678 HAVEMD5=1 # Always assume we have a MD5SUM
679 MD5=`grep " $FILE" MD5SUMS`
680 if [ $? -ne 0 ]; then
681 set -e
682 gs_wrt_message "$FILE not found in MD5SUMS"
683 # IMPORTANT:: There is no MD5SUM for this file.
684 HAVEMD5=0
685 fi
686
687 set -e
688 # If the file exists in the archive copy it to the
689 # $BUILDDIR/sources dir. MD5SUM will be validated later.
690 if [ ! -z ${SRC_ARCHIVE} ] &&
691 [ -d ${SRC_ARCHIVE} ] &&
692 [ -f ${SRC_ARCHIVE}/$FILE ]; then
693 cp ${SRC_ARCHIVE}/$FILE .
694 echo "$FILE: -- copied from $SRC_ARCHIVE"
695 fromARCHIVE=1
696 else
697 echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}"
698 fromARCHIVE=0
699 # If the file does not exist in /sources download a fresh one
700 if [ ! -f $FILE ] ; then
701 if ! wget $URL1 && ! wget $URL2 ; then
702 gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
703 continue
704 fi
705 fi
706 fi
707
708 # Is there a MD5SUM to validate the file against.
709 if [[ "$HAVEMD5" = "1" ]] ; then
710 # IF the md5sum does not match the existing files
711 if ! echo "$MD5" | md5sum -c - >/dev/null ; then
712 [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
713 [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
714 # Remove the old file and download a new one
715 rm -fv $FILE
716 # Force generation of MD5SUM and storage in SRC_ARCHIVE
717 fromARCHIVE=0; HAVEMD5=0
718 # Try and retrieve the file
719 # I swap server priority to allow for corruption on the primary site
720 if ! wget -N $URL1 && ! wget -N $URL2 ; then
721 gs_wrt_message "$FILE not found on the servers.. SKIPPING"
722 continue
723 fi
724 fi
725 fi
726
727 # Validate the MD5SUM one last time
728 if [[ "$HAVEMD5" = "1" ]] && ! echo "$MD5" | md5sum -c - >/dev/null ; then
729 gs_wrt_message "$FILE does not match MD5SUMS value"
730 fi
731
732 # Generate a fresh MD5SUM for this file
733 if [[ "$HAVEMD5" = "0" ]] ; then
734 echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
735 echo "NEW MD5SUM $(md5sum $FILE)" >> MISSING_FILES.DMP
736 fi
737
738 # Good or bad we write the original md5sum to a file
739 echo "$MD5" >> MD5SUMS-$VERSION
740
741 # Copy the freshly downloaded file
742 # to the source archive.
743 if [ ! -z ${SRC_ARCHIVE} ] &&
744 [ -d ${SRC_ARCHIVE} ] &&
745 [ -w ${SRC_ARCHIVE} ] &&
746 [ "$fromARCHIVE" = "0" ] ; then
747 echo "Storing file:<$FILE> in the package archive"
748 cp -f $FILE ${SRC_ARCHIVE}
749 fi
750
751 done
752
753 if [[ -s MISSING_FILES.DMP ]]; then
754 echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
755 # Do not allow the automatic exection of the Makefile.
756 echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
757 RUNMAKE=0
758 fi
759}
760
761
762
763#----------------------------#
764get_sourcesX() { # Download file, write name to MISSING_FILES.DMP if an error
765#----------------------------#
766 local saveIFS=$IFS
767 local IFS line URL1 URL2 FILE MD5 NOMD5
768
769 # Test if the packages must be downloaded
770 if [ "$GETPKG" = "1" ] ; then
771
772 [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
773 cd $BUILDDIR/sources
774 [[ -f MD5SUMS ]] && rm MD5SUMS
775 [[ -f MD5SUMS-$VERSION ]] && rm MD5SUMS-$VERSION
776 [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP
777 [[ -f urls.lst ]] && rm urls.lst
778
779 # Download a fresh MD5SUMS file
780 wget $SERVER/pub/lfs/conglomeration/MD5SUMS
781
782 # Generate URLs file
783 create_urls
784
785 IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
786 for line in `cat urls.lst`; do
787 IFS=$saveIFS # Restore the system defaults
788
789 # Skip some packages if they aren't needed
790 case $line in
791 */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* )
792 [[ "$TEST" = "0" ]] && continue
793 ;;
794 */vim-*-lang* )
795 [[ "$VIMLANG" = "0" ]] && continue
796 ;;
797 esac
798
799 URL1=`echo $line | cut -d" " -f1` # Upstream URL
800 URL2=`echo $line | cut -d" " -f2` # Fallback URL
801 FILE=`basename $URL2` # File name
802
803 # If the file exists in the archive copy it to the $BUILDDIR/sources dir
804 # MD5SUM is assumed to be correct from previous download
805 if [ ! -z ${SRC_ARCHIVE} ] &&
806 [ -d ${SRC_ARCHIVE} ] &&
807 [ -f ${SRC_ARCHIVE}/$FILE ]; then
808 cp ${SRC_ARCHIVE}/$FILE .
809 echo "$FILE: -- copied from $SRC_ARCHIVE" && continue
810 fi
811
812 # Find the md5 sum for this package.
813 set +e
814 MD5=`grep " $FILE" MD5SUMS`
815 if [ $? -ne 0 ]; then
816 set -e
817 echo "${RED}$FILE not found in MD5SUMS${OFF}"
818 echo "$FILE not found in MD5SUMS" >> MISSING_FILES.DMP
819 NOMD5=1
820 fi
821 set -e
822
823 if [ ! -f $FILE ] ; then
824 wget $URL1 || wget $URL2 || (echo "${RED}$FILE not found on the servers${OFF}" &&
825 echo "$FILE not found on the servers" >> MISSING_FILES.DMP )
826 elif [[ "$NOMD5" != "1" ]] && ! echo "$MD5" | md5sum -c - ; then
827 wget -N $URL1 || wget -N $URL2 || ( echo "${RED}$FILE not found on the servers${OFF}" &&
828 echo "$FILE not found on the servers" >> MISSING_FILES.DMP )
829 fi
830
831 if [[ "$NOMD5" != "1" ]] && ! echo "$MD5" | md5sum -c - ; then
832 echo "${RED}$FILE not match MD5SUMS value${OFF}"
833 echo "$FILE not match MD5SUMS value" >> MISSING_FILES.DMP
834 fi
835 echo `grep "$MD5" MD5SUMS` >> MD5SUMS-$VERSION
836
837 # Copy the freshly downloaded file
838 # to the source archive.
839 if [ ! -z ${SRC_ARCHIVE} ] &&
840 [ -d ${SRC_ARCHIVE} ] &&
841 [ -w ${SRC_ARCHIVE} ] &&
842 [ ! -f ${SRC_ARCHIVE}/$FILE ] ; then
843 echo "Store file:<$FILE> in package archive"
844 cp -v $FILE ${SRC_ARCHIVE}
845 fi
846
847 done
848
849 if [[ -s MISSING_FILES.DMP ]]; then
850 echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n\n"
851 fi
852
853 fi
854}
855
856
857#----------------------------#
858create_urls() { #
859#----------------------------#
860 cd $JHALFSDIR
861
862 case ${PROGNAME} in
863 clfs)
864 echo -n "Creating CLFS <${ARCH}> specific URLs file"
865 xsltproc --nonet --xinclude \
866 --stringparam server $SERVER \
867 -o $BUILDDIR/sources/urls.lst urls.xsl \
868 $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
869 echo " ...OK"
870 ;;
871 hlfs)
872 echo -n "Creating HLFS <${MODEL}> specific URLs file"
873 xsltproc --nonet --xinclude \
874 --stringparam server $SERVER \
875 --stringparam model $MODEL \
876 -o $BUILDDIR/sources/urls.lst urls.xsl \
877 $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1
878 echo " ...OK"
879 ;;
880 lfs)
881 echo -n "Creating LFS specific URLs file"
882 xsltproc --nonet --xinclude \
883 --stringparam server $SERVER \
884 -o ../sources/urls.lst urls.xsl \
885 $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1
886 echo " ...OK"
887 ;;
888 esac
889
890 cd $BUILDDIR/sources
891}
Note: See TracBrowser for help on using the repository browser.