source: common/common-functions@ 26d25e8

1.0 2.3 2.3.x 2.4 ablfs ablfs-more legacy new_features trunk
Last change on this file since 26d25e8 was 296ae69, checked in by Jeremy Huntwork <jhuntwork@…>, 19 years ago

Changed order of download loacations; prefer $SERVER to upstream

  • Property mode set to 100644
File size: 26.7 KB
Line 
1#!/bin/bash
2
3# $Id$
4
5set +e
6
7# VT100 colors
8declare -r BLACK=$'\e[1;30m'
9declare -r DK_GRAY=$'\e[0;30m'
10
11declare -r RED=$'\e[31m'
12declare -r GREEN=$'\e[32m'
13declare -r YELLOW=$'\e[33m'
14declare -r BLUE=$'\e[34m'
15declare -r MAGENTA=$'\e[35m'
16declare -r CYAN=$'\e[36m'
17declare -r WHITE=$'\e[37m'
18
19declare -r OFF=$'\e[0m'
20declare -r BOLD=$'\e[1m'
21declare -r REVERSE=$'\e[7m'
22declare -r HIDDEN=$'\e[8m'
23
24declare -r tab_=$'\t'
25declare -r nl_=$'\n'
26
27declare -r DD_BORDER="${BOLD}==============================================================================${OFF}"
28declare -r SD_BORDER="${BOLD}------------------------------------------------------------------------------${OFF}"
29declare -r STAR_BORDER="${BOLD}******************************************************************************${OFF}"
30
31# bold yellow > < pair
32declare -r R_arrow=$'\e[1;33m>\e[0m'
33declare -r L_arrow=$'\e[1;33m<\e[0m'
34
35
36usage() {
37cat <<- -EOF-
38${DD_BORDER}
39${BOLD}
40 Usage: $0 ${BOLD}[OPTION]
41
42Options:
43${BOLD} -h, --help${OFF}
44 print this help, then exit
45
46${BOLD} -V, --version${OFF}
47 print version information, then exit
48
49${BOLD} -B, --book VER${OFF}
50 checkout VER version of the book. Supported versions at this time are:
51 dev* | trunk | SVN aliases for Development {C,H,B}LFS
52
53${BOLD} -D --directory DIR${OFF}
54 use DIR directory for building ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF}; all files jhalfs produces
55 will be in the directory DIR/jhalfs.
56 The current setting for BUILDDIR is "$BUILDDIR"
57
58${BOLD} -G, --get-packages${OFF}
59 download the packages and patches. This assumes that the server declared
60 in the configuration file has the proper packages and patches for the
61 book version being processed.
62
63${BOLD} -T, --testsuites N ${OFF}
64 Run test suites [0-3]
65 0 = none
66 1 = only chapter06 Glibc, GCC and Binutils testsuites
67 2 = all chapter06 testsuites (in BLFS, alias to 1)
68 3 = all chapter05 and chapter06 testsuites
69 In BLFS, 2 and 3 are aliases to 1
70 In CLFS, 3 is an alias to 2
71
72${BOLD} -W, --working-copy DIR${OFF}
73 use the local working copy placed in DIR as the $(echo $PROGNAME | tr [a-z] [A-Z]) book
74-EOF-
75
76[[ ${PROGNAME} != "blfs" ]] &&
77cat <<- -EOF-
78
79${BOLD} -C, --comparasion TYPE${OFF}
80 do iterative comparison analysis. That will take several time due that
81 the final system packages wil be rebuilded the times defined in the
82 ITERATIONS config option.
83 Analisys types allowed are:
84 ICA = do the ICA analysis designed by Greg Schafer
85 farce = do the farce analysis designed by Ken Moffat
86 both = perfom both ICA and farce analysis
87
88${BOLD} -F, --fstab FILE${OFF}
89 use FILE as the /etc/fstab file for the ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF} system. If not specified,
90 a default /etc/fstab file with dummy values is created.
91
92${BOLD} -K, --kernel-config FILE${OFF}
93 use the kernel configuration file specified in FILE to build the kernel.
94 if the file is not found, or if not specified, the kernel build is skipped.
95
96${BOLD} -M, --run-make${OFF}
97 run make on the generated Makefile
98
99${BOLD} -R --rebuild${OFF}
100 clean the build directory before to perfom any other task. The directory
101 is cleaned only if it was populated by a previous jhalfs run.
102-EOF-
103
104[[ ${PROGNAME} = "clfs" ]] &&
105cat <<- -EOF-
106
107${BOLD} -A, --arch ARCH ${OFF}
108 Select the TARGET architecture, valid selections are:
109 32bit builds
110 x86, i486, i586, ppc, mips, mipsel, sparc
111 64bit builds
112 x86_64-64, mips64-64, mipsel64-64, sparc64-64, alpha
113 64bit multi-lib
114 x86_64, mips64, mipsel64, sparc64, ppc64
115
116${BOLD} --boot-config FILE ${OFF}
117 The configuration file for the bootstrap kernel if method=boot
118
119${BOLD} --method BUILDMETHOD ${OFF}
120 Select the build method, chroot or boot
121-EOF-
122
123[[ ${PROGNAME} = "hlfs" ]] &&
124cat <<- -EOF-
125
126${BOLD} --model STYLE ${OFF}
127 Select the library model for the HLFS system
128 Valid choices are: glibc or uclibc
129-EOF-
130
131[[ ${PROGNAME} = "blfs" ]] &&
132cat <<- -EOF-
133
134${BOLD} --dependencies 0/1/2${OFF}
135 add dependencies of type TYPE to the build tree.
136 If not set, both required and recommended are used.
137
138 Possible values are:
139 0 - required only required dependecies are used
140 1 - recommended both required a recommended dependencies are used
141 2 - optional all dependencies are used
142-EOF-
143
144cat <<- -EOF-
145${DD_BORDER}
146-EOF-
147 exit
148}
149
150version="
151${BOLD} \"jhalfs\"${OFF} builder tool (development) \$Rev$
152\$Date$
153
154${BOLD} \"${PROGNAME}\"${OFF} script module
155
156Written by George Boudreau,
157 Manuel Canales Esparcia,
158 Jeremy Huntwork
159
160This program is published under the ${BOLD}Gnu General Public License, Version 2.${OFF}
161"
162
163
164no_empty_builddir() {
165 'clear'
166cat <<- -EOF-
167${DD_BORDER}
168
169${tab_}${tab_}${BOLD}${RED}W A R N I N G${OFF}
170 Looks like the \$BUILDDIR directory contains subdirectories
171 from a previous HLFS build.
172
173 Please format the partition mounted on \$BUILDDIR or set
174 a different build directory before running jhalfs.
175${OFF}
176${DD_BORDER}
177-EOF-
178 exit
179}
180
181
182help="${nl_}Try '$0 --help' for more information."
183
184
185exit_missing_arg="\
186echo \"Option '\$1' requires an argument\" >&2
187echo \"\$help\" >&2
188exit 1"
189
190no_dl_client="\
191echo \"Could not find a way to download the ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF} sources.\" >&2
192echo \"Attempting to continue.\" >&2"
193
194HEADER="# This file is automatically generated by jhalfs
195# DO NOT EDIT THIS FILE MANUALLY
196#
197# Generated on `date \"+%F %X %Z\"`"
198
199
200
201
202
203#----------------------------------#
204wrt_target() { # Create target and initialize log file
205#----------------------------------#
206 local i=$1
207 local PREV=$2
208(
209cat << EOF
210
211$i: $PREV
212 @\$(call echo_message, Building)
213 @echo -e "\n\`date\`\n\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >logs/$this_script
214EOF
215) >> $MKFILE.tmp
216}
217
218
219#----------------------------------#
220wrt_unpack() { # Unpack and set 'ROOT' var
221#----------------------------------#
222 local FILE=$1
223(
224cat << EOF
225 @\$(call unpack,$FILE)
226 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
227 echo "export PKGDIR=\$(MOUNT_PT)\$(SRC)/\$\$ROOT" > envars && \\
228 chown -R lfs \$(MOUNT_PT)\$(SRC)/\$\$ROOT
229EOF
230) >> $MKFILE.tmp
231}
232
233
234#----------------------------------#
235wrt_unpack2() { #
236#----------------------------------#
237 local FILE=$1
238(
239cat << EOF
240 @\$(call unpack2,$FILE)
241 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
242 echo "export PKGDIR=\$(SRC)/\$\$ROOT" > envars
243EOF
244) >> $MKFILE.tmp
245}
246
247
248#=============================#
249wrt_unpack3() { #
250#=============================#
251 local FILE=$1
252(
253cat << EOF
254 @\$(call unpack3,$FILE)
255 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
256 echo "export PKGDIR=\$(SRC)/\$\$ROOT" > envars
257EOF
258) >> $MKFILE.tmp
259}
260
261
262
263#----------------------------------#
264wrt_target_vars() { # Target vars for hlfs (cross-build method)
265#----------------------------------#
266(
267cat << EOF
268 @echo "export target=$(uname -m)-${TARGET}" >> envars && \\
269 echo "export ldso=/lib/${LOADER}" >> envars
270EOF
271) >> $MKFILE.tmp
272
273}
274
275
276#----------------------------------#
277wrt_run_as_su() { # Execute script inside time { }, footer to log file
278#----------------------------------#
279 local this_script=$1
280 local file=$2
281(
282cat << EOF
283 @( time { su - lfs -c "source /home/lfs/.bashrc && $JHALFSDIR/${PROGNAME}-commands/$file" >>logs/$this_script 2>&1 ; } ) 2>>logs/$this_script && \\
284 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/$this_script
285EOF
286) >> $MKFILE.tmp
287}
288
289
290#----------------------------------#
291wrt_run_as_root() { # Some scripts must be run as root..
292#----------------------------------#
293 local this_script=$1
294 local file=$2
295(
296cat << EOF
297 @( time { export LFS=\$(MOUNT_PT) && ${PROGNAME}-commands/$file >>logs/$this_script 2>&1 ; } ) 2>>logs/$this_script && \\
298 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/$this_script
299EOF
300) >> $MKFILE.tmp
301}
302
303
304#=============================#
305wrt_run_as_root2() { #
306#=============================#
307 local this_script=$1
308 local file=$2
309(
310cat << EOF
311 @( time { source envars && ${PROGNAME}-commands/$file >>logs/$this_script 2>&1 ; } ) 2>>logs/$this_script && \\
312 echo -e "\nKB: \`du -skx --exclude=jhalfs \`\n" >>logs/$this_script
313EOF
314) >> $MKFILE.tmp
315}
316
317
318
319#----------------------------------#
320wrt_run_as_chroot1() { #
321#----------------------------------#
322 local this_script=$1
323 local file=$2
324(
325 cat << EOF
326 @( time { \$(CHROOT1) 'cd /jhalfs && source envars && /jhalfs/${PROGNAME}-commands/$file >>/jhalfs/logs/${this_script} 2>&1' ; } ) 2>>logs/$this_script && \\
327 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/${this_script}
328EOF
329) >> $MKFILE.tmp
330}
331
332
333#----------------------------------#
334wrt_run_as_chroot2() { #
335#----------------------------------#
336 local this_script=$1
337 local file=$2
338(
339cat << EOF
340 @( time { \$(CHROOT2) 'cd /jhalfs && source envars && /jhalfs/${PROGNAME}-commands/$file >>/jhalfs/logs/${this_script} 2>&1' ; } ) 2>>logs/$this_script && \\
341 echo -e "\nKB: \`du -skx --exclude=jhalfs \$(MOUNT_PT)\`\n" >>logs/${this_script}
342EOF
343) >> $MKFILE.tmp
344}
345
346
347#----------------------------------#
348wrt_copy_fstab() { #
349#----------------------------------#
350 local i=$1
351(
352 cat << EOF
353 @cp -v $FSTAB \$(MOUNT_PT)/etc/fstab >>logs/$i 2>&1
354EOF
355) >> $MKFILE.tmp
356}
357
358#----------------------------------#
359wrt_copy_fstab2() { #
360#----------------------------------#
361 local i=$1
362(
363 cat << EOF
364 @cp -v /sources/fstab /etc/fstab >>logs/$i 2>&1
365EOF
366) >> $MKFILE.tmp
367}
368
369
370#----------------------------------#
371wrt_remove_build_dirs() { #
372#----------------------------------#
373 local name=$1
374(
375cat << EOF
376 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
377 rm -r \$(MOUNT_PT)\$(SRC)/\$\$ROOT && \\
378 if [ -e \$(MOUNT_PT)\$(SRC)/$name-build ]; then \\
379 rm -r \$(MOUNT_PT)\$(SRC)/$name-build; \\
380 fi;
381EOF
382) >> $MKFILE.tmp
383}
384
385
386#----------------------------------#
387wrt_remove_build_dirs2() { #
388#----------------------------------#
389 local name=$1
390(
391cat << EOF
392 @ROOT=\`head -n1 /tmp/unpacked | sed 's@^./@@;s@/.*@@'\` && \\
393 rm -r \$(SRC)/\$\$ROOT && \\
394 if [ -e \$(SRC)/$name-build ]; then \\
395 rm -r \$(SRC)/$name-build; \\
396 fi;
397EOF
398) >> $MKFILE.tmp
399}
400
401
402#----------------------------------#
403wrt_report() { #
404#----------------------------------#
405(
406cat << EOF
407
408create-sbu_du-report: $PREV
409 @\$(call echo_message, Building)
410 @./create-sbu_du-report.sh logs $VERSION
411 @\$(call echo_report,$VERSION-SBU_DU-$(date --iso-8601).report)
412 @touch \$@
413EOF
414) >> $MKFILE.tmp
415
416chapter789="$chapter789 create-sbu_du-report"
417}
418
419
420#----------------------------#
421run_make() {
422#----------------------------#
423 # Test if make must be run.
424 if [ "$RUNMAKE" = "1" ] ; then
425 # Test to make sure we're running the build as root
426 if [ "$UID" != "0" ] ; then
427 echo "You must be logged in as root to successfully build the system."
428 exit 1
429 fi
430 # Build the system
431 if [ -e $MKFILE ] ; then
432 echo -ne "Building the system...\n"
433 cd $JHALFSDIR && make
434 echo -ne "done\n"
435 fi
436 fi
437}
438
439
440#----------------------------#
441clean_builddir() {
442#----------------------------#
443 # Test if the clean must be done.
444 if [ "$CLEAN" = "1" ] ; then
445 # Test to make sure we're running the clean as root
446 if [ "$UID" != "0" ] ; then
447 echo "You must be logged in as root to clean the build directory."
448 exit 1
449 fi
450 # Test to make sure that the build directory was populated by jhalfs
451 if [ ! -d $JHALFSDIR ] || [ ! -d $BUILDDIR/sources ] ; then
452 echo "Looks like $BUILDDIR was not populated by a previous jhalfs run."
453 exit 1
454 else
455 # Clean the build directory
456 echo -ne "Cleaning $BUILDDIR...\n"
457 rm -rf $BUILDDIR/{bin,boot,dev,etc,home,lib,media,mnt,opt,proc,root,sbin,srv,sys,tmp,tools,cross-tools,usr,var}
458 echo -ne "Cleaning $JHALFSDIR...\n"
459 rm -rf $JHALFSDIR/{0*,1*,envars,sources-dir,*commands,logs,Makefile,*.xsl,makefile-functions,packages,patches}
460 echo -ne "Cleaning remainig extracted sources in $BUILDDIR/sources...\n"
461 rm -rf `find $BUILDDIR/sources/* -maxdepth 0 -type d`
462 echo -ne "done\n"
463 fi
464 fi
465}
466
467#----------------------------#
468get_book() {
469#----------------------------#
470 cd $JHALFSDIR
471
472 if [ -z $WC ] ; then
473 # Check for Subversion instead of just letting the script hit 'svn' and fail.
474 test `type -p svn` || eval "echo \"This feature requires Subversion.\"
475 exit 1"
476 echo -n "Downloading the $PROGNAME document, $LFSVRS version... "
477
478 case $PROGNAME in
479 lfs) svn_root="LFS" ;;
480 hlfs) svn_root="HLFS" ;;
481 clfs) svn_root="cross-lfs" ;;
482 blfs) svn_root="BLFS" ;;
483 *) echo "BOOK not defined in function <get_book>"
484 exit 1 ;;
485 esac
486 # Grab a fresh book if it's missing, otherwise, update it from the
487 # repo. If we've already extracted the commands, move on to getting the
488 # sources.
489 if [ -d ${PROGNAME}-$LFSVRS ] ; then
490 cd ${PROGNAME}-$LFSVRS
491 if LC_ALL=C svn up | grep -q At && test -d $JHALFSDIR/${PROGNAME}-commands && \
492 test -f $JHALFSDIR/packages ; then
493 echo -ne "done\n"
494 # Set the canonical book version
495 cd $JHALFSDIR
496 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
497 get_sources
498 else
499 echo -ne "done\n"
500 # Set the canonical book version
501 cd $JHALFSDIR
502 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
503 extract_commands
504 fi
505 else
506 case $LFSVRS in
507 development)
508 svn co $SVN/${svn_root}/trunk/BOOK ${PROGNAME}-$LFSVRS >>$LOGDIR/$LOG 2>&1 ;;
509 esac
510 echo -ne "done\n"
511 # Set the canonical book version
512 cd $JHALFSDIR
513 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
514 extract_commands
515 fi
516 else
517 echo -ne "Using $BOOK as book's sources ...\n"
518 # Set the canonical book version
519 cd $JHALFSDIR
520 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
521 extract_commands
522 fi
523}
524
525#----------------------------#
526extract_commands() { #
527#----------------------------#
528
529 # Check for libxslt instead of just letting the script hit 'xsltproc' and fail.
530 test `type -p xsltproc` || eval "echo \"This feature requires libxslt.\"
531 exit 1"
532
533 cd $JHALFSDIR
534 VERSION=`grep "ENTITY version " $BOOK/general.ent | sed 's@<!ENTITY version "@@;s@">@@'`
535
536 # Start clean
537 if [ -d ${PROGNAME}-commands ]; then
538 rm -rf ${PROGNAME}-commands
539 mkdir -v ${PROGNAME}-commands
540 fi
541 echo -n "Extracting commands for"
542
543 # Dump the commands in shell script form from the HLFS book.
544 case ${PROGNAME} in
545 clfs)
546 echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture"
547 xsltproc --nonet \
548 --xinclude \
549 --stringparam method $METHOD \
550 --stringparam testsuite $TEST \
551 --stringparam vim-lang $VIMLANG \
552 --stringparam timezone $TIMEZONE \
553 --stringparam page $PAGE \
554 --stringparam lang $LANG \
555 --stringparam keymap $KEYMAP \
556 -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1
557 ;;
558 hlfs)
559 echo -n " ${L_arrow}${BOLD}$MODEL${R_arrow} HLFS libc implementation"
560 xsltproc --nonet \
561 --xinclude \
562 --stringparam model $MODEL \
563 --stringparam testsuite $TEST \
564 --stringparam timezone $TIMEZONE \
565 --stringparam page $PAGE \
566 --stringparam lang $LANG \
567 --stringparam lc_all $LC_ALL \
568 --stringparam keymap $KEYMAP \
569 --stringparam grsecurity_host $GRSECURITY_HOST \
570 -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1
571 ;;
572 lfs)
573 echo -n " ${L_arrow}${BOLD}LFS${R_arrow} build"
574 xsltproc --nonet \
575 --xinclude \
576 --stringparam testsuite $TEST \
577 --stringparam vim-lang $VIMLANG \
578 --stringparam timezone $TIMEZONE \
579 --stringparam page $PAGE \
580 --stringparam lang $LANG \
581 -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1
582 ;;
583 blfs)
584 echo -n " ${L_arrow}${BOLD}BLFS${R_arrow} build"
585 xsltproc --nonet \
586 --xinclude \
587 --stringparam testsuite $TEST \
588 --stringparam server $SERVER \
589 -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1
590 ;;
591 *) exit 1 ;;
592 esac
593
594 echo " ...OK"
595
596 # Make the scripts executable.
597 chmod -R +x $JHALFSDIR/${PROGNAME}-commands
598
599 # Create the packages file. We need it for proper Makefile creation
600 rm -f packages
601 case "${PROGNAME}" in
602 clfs)
603 echo -n "Creating <${PROGNAME}> specific packages file"
604 grep "\-version " $BOOK/packages.ent | sed -e 's@<!ENTITY @@' \
605 -e 's@">@"@' \
606 -e '/generic/d' > packages
607 echo " ...OK"
608 ;;
609
610 hlfs)
611 echo -n "Creating <${PROGNAME}> specific packages file"
612 grep "\-version" $BOOK/general.ent | sed -e 's@<!ENTITY @@' \
613 -e 's@">@"@' \
614 -e '/generic/d' >> packages
615 echo " ...OK"
616 ;;
617
618 lfs)
619 echo -n "Creating <${PROGNAME}> specific packages file"
620 grep "\-version" $BOOK/general.ent | sed -e 's@<!ENTITY @@' \
621 -e 's@">@"@' \
622 -e '/generic/d' >> packages
623 echo " ...OK"
624 ;;
625 esac
626
627 # Done. Moving on...
628 get_sources
629}
630
631#----------------------------#
632get_sources() { # Download file, write name to MISSING_FILES.DMP if an error
633#----------------------------#
634 local saveIFS=$IFS
635 local IFS line URL1 URL2 FILE MD5 HAVEMD5 fromARCHIVE
636
637 # Test if the packages must be downloaded
638 [ ! "$GETPKG" = "1" ] && return
639
640 gs_wrt_message(){
641 echo "${RED}$1${OFF}"
642 echo "$1" >> MISSING_FILES.DMP
643 }
644 # Housekeeping
645 [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
646 cd $BUILDDIR/sources
647 [[ -f MD5SUMS ]] && rm MD5SUMS
648 [[ -f MD5SUMS-$VERSION ]] && rm MD5SUMS-$VERSION
649 [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP
650 [[ -f urls.lst ]] && rm urls.lst
651
652 # Download a fresh MD5SUMS file
653 wget $SERVER/pub/lfs/conglomeration/MD5SUMS
654
655 # Generate URLs file
656 create_urls
657
658 IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
659 for line in `cat urls.lst`; do
660 IFS=$saveIFS # Restore the system defaults
661
662 # Skip some packages if they aren't needed
663 case $line in
664 */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* )
665 [[ "$TEST" = "0" ]] && continue
666 ;;
667 */vim-*-lang* )
668 [[ "$VIMLANG" = "0" ]] && continue
669 ;;
670 esac
671
672 # Locations
673 URL1=`echo $line | cut -d" " -f2` # Preferred URL
674 URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL
675 FILE=`basename $URL2` # File name
676
677 # Find the md5 sum for this package.
678 set +e
679 HAVEMD5=1 # Always assume we have a MD5SUM
680 MD5=`grep " $FILE" MD5SUMS`
681 if [ $? -ne 0 ]; then
682 set -e
683 gs_wrt_message "$FILE not found in MD5SUMS"
684 # IMPORTANT:: There is no MD5SUM for this file.
685 HAVEMD5=0
686 fi
687
688 set -e
689 # If the file exists in the archive copy it to the
690 # $BUILDDIR/sources dir. MD5SUM will be validated later.
691 if [ ! -z ${SRC_ARCHIVE} ] &&
692 [ -d ${SRC_ARCHIVE} ] &&
693 [ -f ${SRC_ARCHIVE}/$FILE ]; then
694 cp ${SRC_ARCHIVE}/$FILE .
695 echo "$FILE: -- copied from $SRC_ARCHIVE"
696 fromARCHIVE=1
697 else
698 echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}"
699 fromARCHIVE=0
700 # If the file does not exist in /sources download a fresh one
701 if [ ! -f $FILE ] ; then
702 if ! wget $URL1 && ! wget $URL2 ; then
703 gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
704 continue
705 fi
706 fi
707 fi
708
709 # Is there a MD5SUM to validate the file against.
710 if [[ "$HAVEMD5" = "1" ]] ; then
711 # IF the md5sum does not match the existing files
712 if ! echo "$MD5" | md5sum -c - >/dev/null ; then
713 [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
714 [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
715 # Remove the old file and download a new one
716 rm -fv $FILE
717 # Force generation of MD5SUM and storage in SRC_ARCHIVE
718 fromARCHIVE=0; HAVEMD5=0
719 # Try and retrieve the file
720 # I swap server priority to allow for corruption on the primary site
721 if ! wget -N $URL1 && ! wget -N $URL2 ; then
722 gs_wrt_message "$FILE not found on the servers.. SKIPPING"
723 continue
724 fi
725 fi
726 fi
727
728 # Validate the MD5SUM one last time
729 if [[ "$HAVEMD5" = "1" ]] && ! echo "$MD5" | md5sum -c - >/dev/null ; then
730 gs_wrt_message "$FILE does not match MD5SUMS value"
731 fi
732
733 # Generate a fresh MD5SUM for this file
734 if [[ "$HAVEMD5" = "0" ]] ; then
735 echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
736 echo "NEW MD5SUM $(md5sum $FILE)" >> MISSING_FILES.DMP
737 fi
738
739 # Good or bad we write the original md5sum to a file
740 echo "$MD5" >> MD5SUMS-$VERSION
741
742 # Copy the freshly downloaded file
743 # to the source archive.
744 if [ ! -z ${SRC_ARCHIVE} ] &&
745 [ -d ${SRC_ARCHIVE} ] &&
746 [ -w ${SRC_ARCHIVE} ] &&
747 [ "$fromARCHIVE" = "0" ] ; then
748 echo "Storing file:<$FILE> in the package archive"
749 cp -f $FILE ${SRC_ARCHIVE}
750 fi
751
752 done
753
754 if [[ -s MISSING_FILES.DMP ]]; then
755 echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
756 # Do not allow the automatic exection of the Makefile.
757 echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
758 RUNMAKE=0
759 fi
760}
761
762
763
764#----------------------------#
765get_sourcesX() { # Download file, write name to MISSING_FILES.DMP if an error
766#----------------------------#
767 local saveIFS=$IFS
768 local IFS line URL1 URL2 FILE MD5 NOMD5
769
770 # Test if the packages must be downloaded
771 if [ "$GETPKG" = "1" ] ; then
772
773 [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
774 cd $BUILDDIR/sources
775 [[ -f MD5SUMS ]] && rm MD5SUMS
776 [[ -f MD5SUMS-$VERSION ]] && rm MD5SUMS-$VERSION
777 [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP
778 [[ -f urls.lst ]] && rm urls.lst
779
780 # Download a fresh MD5SUMS file
781 wget $SERVER/pub/lfs/conglomeration/MD5SUMS
782
783 # Generate URLs file
784 create_urls
785
786 IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
787 for line in `cat urls.lst`; do
788 IFS=$saveIFS # Restore the system defaults
789
790 # Skip some packages if they aren't needed
791 case $line in
792 */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* )
793 [[ "$TEST" = "0" ]] && continue
794 ;;
795 */vim-*-lang* )
796 [[ "$VIMLANG" = "0" ]] && continue
797 ;;
798 esac
799
800 URL1=`echo $line | cut -d" " -f1` # Upstream URL
801 URL2=`echo $line | cut -d" " -f2` # Fallback URL
802 FILE=`basename $URL2` # File name
803
804 # If the file exists in the archive copy it to the $BUILDDIR/sources dir
805 # MD5SUM is assumed to be correct from previous download
806 if [ ! -z ${SRC_ARCHIVE} ] &&
807 [ -d ${SRC_ARCHIVE} ] &&
808 [ -f ${SRC_ARCHIVE}/$FILE ]; then
809 cp ${SRC_ARCHIVE}/$FILE .
810 echo "$FILE: -- copied from $SRC_ARCHIVE" && continue
811 fi
812
813 # Find the md5 sum for this package.
814 set +e
815 MD5=`grep " $FILE" MD5SUMS`
816 if [ $? -ne 0 ]; then
817 set -e
818 echo "${RED}$FILE not found in MD5SUMS${OFF}"
819 echo "$FILE not found in MD5SUMS" >> MISSING_FILES.DMP
820 NOMD5=1
821 fi
822 set -e
823
824 if [ ! -f $FILE ] ; then
825 wget $URL1 || wget $URL2 || (echo "${RED}$FILE not found on the servers${OFF}" &&
826 echo "$FILE not found on the servers" >> MISSING_FILES.DMP )
827 elif [[ "$NOMD5" != "1" ]] && ! echo "$MD5" | md5sum -c - ; then
828 wget -N $URL1 || wget -N $URL2 || ( echo "${RED}$FILE not found on the servers${OFF}" &&
829 echo "$FILE not found on the servers" >> MISSING_FILES.DMP )
830 fi
831
832 if [[ "$NOMD5" != "1" ]] && ! echo "$MD5" | md5sum -c - ; then
833 echo "${RED}$FILE not match MD5SUMS value${OFF}"
834 echo "$FILE not match MD5SUMS value" >> MISSING_FILES.DMP
835 fi
836 echo `grep "$MD5" MD5SUMS` >> MD5SUMS-$VERSION
837
838 # Copy the freshly downloaded file
839 # to the source archive.
840 if [ ! -z ${SRC_ARCHIVE} ] &&
841 [ -d ${SRC_ARCHIVE} ] &&
842 [ -w ${SRC_ARCHIVE} ] &&
843 [ ! -f ${SRC_ARCHIVE}/$FILE ] ; then
844 echo "Store file:<$FILE> in package archive"
845 cp -v $FILE ${SRC_ARCHIVE}
846 fi
847
848 done
849
850 if [[ -s MISSING_FILES.DMP ]]; then
851 echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n\n"
852 fi
853
854 fi
855}
856
857
858#----------------------------#
859create_urls() { #
860#----------------------------#
861 cd $JHALFSDIR
862
863 case ${PROGNAME} in
864 clfs)
865 echo -n "Creating CLFS <${ARCH}> specific URLs file"
866 xsltproc --nonet --xinclude \
867 --stringparam server $SERVER \
868 -o $BUILDDIR/sources/urls.lst urls.xsl \
869 $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
870 echo " ...OK"
871 ;;
872 hlfs)
873 echo -n "Creating HLFS <${MODEL}> specific URLs file"
874 xsltproc --nonet --xinclude \
875 --stringparam server $SERVER \
876 --stringparam model $MODEL \
877 -o $BUILDDIR/sources/urls.lst urls.xsl \
878 $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1
879 echo " ...OK"
880 ;;
881 lfs)
882 echo -n "Creating LFS specific URLs file"
883 xsltproc --nonet --xinclude \
884 --stringparam server $SERVER \
885 -o ../sources/urls.lst urls.xsl \
886 $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1
887 echo " ...OK"
888 ;;
889 esac
890
891 cd $BUILDDIR/sources
892}
Note: See TracBrowser for help on using the repository browser.