source: LFS/master.sh@ cff36a7

ablfs-more trunk
Last change on this file since cff36a7 was cacc6c5, checked in by Pierre Labastie <pierre.labastie@…>, 12 months ago

Allow users to pass a cpuset

If using a new book, MAKEFLAGS is set to -j$(nproc), so that if
we want to easure timings with 4 processors and our computer has
more than that, we need to restrict the number of processors.
Jhalfs allows to do that using th cpuset cgroup controller, so that
it only restrits the make job and not the whole machine. For that
use a small scripts written by Xi Ruoyao that restarts make this
the cpuset set.
  • Property mode set to 100644
File size: 17.3 KB
Line 
1#!/bin/bash
2
3###################################
4### FUNCTIONS ###
5###################################
6
7
8#############################################################
9
10
11#-------------------------#
12chapter_targets() { #
13#-------------------------#
14# $1 is the chapter number. Pad it with 0 to the left to obtain a 2-digit
15# number:
16 printf -v dir chapter%02d $1
17
18# $2 contains the build number if rebuilding for ICA
19 if [[ -z "$2" ]] ; then
20 local N=""
21 else
22 local N=-build_$2
23 local CHROOT_TGT=""
24 mkdir ${dir}$N
25 cp ${dir}/* ${dir}$N
26 for script in ${dir}$N/* ; do
27 # Overwrite existing symlinks, files, and dirs
28 sed -e 's/ln *-sv/&f/g' \
29 -e 's/mv *-v/&f/g' \
30 -e 's/mkdir *-v/&p/g' -i ${script}
31 # Rename the scripts
32 mv ${script} ${script}$N
33 done
34 # Remove Bzip2 binaries before make install (LFS-6.2 compatibility)
35 sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i ${dir}$N/*-bzip2$N
36 # Remove openssl-<version> from /usr/share/doc (LFS-9.x), because
37 # otherwise the mv command creates an openssl directory.
38 sed -e 's@mv -v@rm -rfv /usr/share/doc/openssl-*\n&@' \
39 -i ${dir}$N/*-openssl$N
40 fi
41
42 echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter $1$N${R_arrow}"
43
44 for file in ${dir}$N/* ; do
45 # Keep the script file name
46 this_script=`basename $file`
47
48 # Some scripts need peculiar actions:
49 # - Stripping at the end of system build: lfs.xsl does not generate
50 # correct commands if the user does not want to strip, so skip it
51 # in this case
52 # - do not reinstall linux-headers when rebuilding
53 # - grub config: must be done manually; skip it
54 # - handle fstab and .config. Skip kernel if .config not supplied
55 case "${this_script}" in
56 *stripping*) [[ "${STRIP}" = "n" ]] && continue ;;
57 *linux-headers*) [[ -n "$N" ]] && continue ;;
58 8*grub) (( nb_chaps == 5 )) && continue ;;
59 10*grub) continue ;;
60 *fstab) [[ -z "${FSTAB}" ]] ||
61 [[ ${FSTAB} == $BUILDDIR/sources/fstab ]] ||
62 cp ${FSTAB} $BUILDDIR/sources/fstab ;;
63 *kernel) [[ -z ${CONFIG} ]] && continue
64 [[ ${CONFIG} == $BUILDDIR/sources/kernel-config ]] ||
65 cp ${CONFIG} $BUILDDIR/sources/kernel-config ;;
66 esac
67 # Grab the name of the target
68 # This is only used to check the name in "opt_override" or "BLACKLIST"
69 name=`echo ${this_script} | sed -e 's@[0-9]\{3,4\}-@@' \
70 -e 's@-pass[0-9]\{1\}@@' \
71 -e 's@-libstdc++@@' \
72 -e 's,'$N',,' \
73 -e 's@-32@@'`
74
75 # Find the name of the tarball and the version of the package
76 # If it doesn't exist, we skip it in iterations rebuilds (except stripping
77 # and revisedchroot, where .a and .la files are removed).
78 pkg_tarball=$(sed -n 's/tar -xf \(.*\)/\1/p' $file)
79 pkg_version=$(sed -n 's/VERSION=\(.*\)/\1/p' $file)
80
81 if [[ "$pkg_tarball" = "" ]] && [[ -n "$N" ]] ; then
82 case "${this_script}" in
83 *stripping*|*cleanup*|*revised*) ;;
84 *) continue ;;
85 esac
86 fi
87
88 # Append the name of the script to a list. The name of the
89 # list is contained in the variable Makefile_target. We adjust this
90 # variable at various points. Note that it is initialized to "SETUP"
91 # in the main function, before calling this function for the first time.
92 case "${this_script}" in
93 *settingenvironment) Makefile_target=LUSER_TGT ;;
94 *changingowner ) Makefile_target=SUDO_TGT ;;
95 *creatingdirs ) Makefile_target=CHROOT_TGT ;;
96 *bootscripts ) Makefile_target=BOOT_TGT ;; # case of sysv book
97 *network ) Makefile_target=BOOT_TGT ;; # case of systemd book
98 esac
99 eval $Makefile_target=\"\$$Makefile_target ${this_script}\"
100
101 #--------------------------------------------------------------------#
102 # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< #
103 #--------------------------------------------------------------------#
104
105 # Drop in the name of the target on a new line, and the previous target
106 # as a dependency. Also call the echo_message function.
107 case $Makefile_target in
108 CHROOT_TGT) CHROOT_wrt_target "${this_script}" "$PREV" "$pkg_version" ;;
109 *) LUSER_wrt_target "${this_script}" "$PREV" "$pkg_version" ;;
110 esac
111
112 # If $pkg_tarball isn't empty, we've got a package...
113 if [ "$pkg_tarball" != "" ] ; then
114 # Touch timestamp file if installed files logs shall be created.
115 # But only for the final install chapter and not when rebuilding it
116 if [ "${INSTALL_LOG}" = "y" ] &&
117 (( 1+nb_chaps <= $1 )) &&
118 [ "x$N" = x ] ; then
119 CHROOT_wrt_TouchTimestamp
120 fi
121 # Always initialize the test log file, since the test instructions may
122 # be "uncommented" by the user
123 case $Makefile_target in
124 CHROOT_TGT) CHROOT_wrt_test_log "${this_script}" "$pkg_version" ;;
125 LUSER_TGT ) LUSER_wrt_test_log "${this_script}" "$pkg_version" ;;
126 esac
127
128 # If using optimizations, write the instructions
129 case "${OPTIMIZE}$1${nb_chaps}${this_script}${REALSBU}" in
130 0* | *binutils-pass1y | 15* | 167* | 177*) ;;
131 *kernel*) ;; # No CFLAGS for kernel
132 *) wrt_optimize "$name" ;;
133 esac
134 # MAKEFLAGS is set even if no optimization now.
135 # For new books, the scripts contain the MAKEFLAGS, so no
136 # need to set them in the envars except if binutils-pass1
137 # and REALSBU=y. For Old books, N_PARALLEL and JH_MAKEFLAGS
138 # are set and we must set the envars.
139 case "${mkf_included}${this_script}${REALSBU}" in
140 *binutils-pass1y)
141 wrt_makeflags "$name" "-j1" "1" ;;
142 n*)
143 wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL" ;;
144 esac
145 fi
146
147# Some scriptlet have a special treatment; otherwise standard
148 case "${this_script}" in
149 *addinguser)
150(
151# /var/lib may already exist and be owned by root if blfs tools
152# have been installed.
153cat << EOF
154 @if [ -f luser-id ]; then \\
155 function useradd() { true; }; \\
156 function groupadd() { true; }; \\
157 export -f useradd groupadd; \\
158 fi; \\
159 export LFS=\$(MOUNT_PT) && \\
160 \$(CMDSDIR)/`dirname $file`/\$@ >> \$(LOGDIR)/\$@ 2>&1; \\
161 \$(PRT_DU) >>logs/\$@
162 @chown \$(LUSER):\$(LGROUP) envars
163 @if [ -d "\$(MOUNT_PT)/var/lib" ]; then \\
164 chown \$(LUSER):\$(LGROUP) \$(MOUNT_PT)/var/lib; \\
165 fi
166 @chmod -R a+wt $JHALFSDIR
167 @chmod a+wt \$(SRCSDIR)
168EOF
169) >> $MKFILE.tmp
170 ;;
171 *settingenvironment)
172(
173cat << EOF
174 @cd && \\
175 function source() { true; } && \\
176 export -f source && \\
177 \$(CMDSDIR)/`dirname $file`/\$@ >> \$(LOGDIR)/\$@ 2>&1 && \\
178 sed 's|/mnt/lfs|\$(MOUNT_PT)|' -i .bashrc && \\
179 echo source $JHALFSDIR/envars >> .bashrc
180 @\$(PRT_DU) >>logs/\$@
181EOF
182) >> $MKFILE.tmp
183 ;;
184 *fstab) if [[ -n "$FSTAB" ]]; then
185 CHROOT_wrt_CopyFstab
186 else
187 CHROOT_wrt_RunAsRoot "$file"
188 fi
189 ;;
190
191 *)
192 # Insert date and disk usage at the top of the log file, the script
193 # run and date and disk usage again at the bottom of the log file.
194 case "${Makefile_target}" in
195 SETUP_TGT | SUDO_TGT) wrt_RunAsRoot "$file" "$pkg_version" ;;
196 LUSER_TGT) LUSER_wrt_RunAsUser "$file" "$pkg_version" ;;
197 CHROOT_TGT | BOOT_TGT) CHROOT_wrt_RunAsRoot "$file" "$pkg_version" ;;
198 esac
199 ;;
200 esac
201
202 # Write installed files log and remove the build directory(ies)
203 # except if the package build fails.
204 if [ "$pkg_tarball" != "" ] ; then
205 if [ "${INSTALL_LOG}" = "y" ] &&
206 (( 1+nb_chaps <= $1 )) &&
207 [ "x${N}" = "x" ] ; then
208 CHROOT_wrt_LogNewFiles "${this_script}"
209 fi
210 fi
211
212 # Include a touch of the target name so make can check
213 # if it's already been made.
214 wrt_touch
215 #
216 #--------------------------------------------------------------------#
217 # >>>>>>>> END OF Makefile ENTRY <<<<<<<< #
218 #--------------------------------------------------------------------#
219
220 # Keep the script file name for Makefile dependencies.
221 PREV=${this_script}
222 # Set "system_build" var for iteration targets
223 if [ -z "$N" ] && (( 1+nb_chaps == $1 )); then
224 system_build="$system_build $this_script"
225 fi
226
227 done # end for file in $dir/*
228 # Set "system_build" when rebuilding: note the CHROOT_TGT is local
229 # in that case.
230 if [ -n "$N" ]; then
231 system_build="$CHROOT_TGT"
232 fi
233}
234
235#----------------------------#
236build_Makefile() { #
237#----------------------------#
238
239 echo "Creating Makefile... ${BOLD}START${OFF}"
240
241 cd "$JHALFSDIR/$COMMANDS"
242
243 # Start with empty files
244 >$MKFILE
245 >$MKFILE.tmp
246
247 # Ensure the first dependency is empty
248 unset PREV
249
250 # We begin with the SETUP target; successive targets will be assigned in
251 # the chapter_targets function.
252 Makefile_target=SETUP_TGT
253
254 # We need to know the chapter numbering, which depends on the version
255 # of the book. Use the number of subdirs to know which version we have
256 chaps=($(echo chapter*))
257 nb_chaps=${#chaps[*]} # 5 if classical version, 7 if new version
258# DEBUG
259# echo chaps: ${chaps[*]}
260# echo nb_chaps: $nb_chaps
261# end DEBUG
262
263 # We need to know if we have an old or a new book (with included MAKEFLAGS)
264 # we grep for MAKEFLAGS in chapter04 to find out.
265 if grep -q MAKEFLAGS chapter04/*; then
266 mkf_included=y
267 else
268 mkf_included=n
269 fi
270
271 # Make a temporary file with all script targets
272 for (( i = 4; i < nb_chaps+4; i++ )); do
273 chapter_targets $i
274 if (( i == nb_chaps )); then : # we have finished temporary tools
275 # Add the save target, if needed
276 [[ "$SAVE_CH5" = "y" ]] && wrt_save_target $Makefile_target
277 fi
278 if (( i == 1+nb_chaps )); then : # we have finished final system
279 # Add the iterations targets, if needed
280 [[ "$COMPARE" = "y" ]] && wrt_compare_targets $i
281 fi
282 done
283 # Add the CUSTOM_TOOLS targets, if needed
284 [[ "$CUSTOM_TOOLS" = "y" ]] && wrt_CustomTools_target
285
286 # Add a header, some variables and include the function file
287 # to the top of the real Makefile.
288 wrt_Makefile_header
289
290 # Add chroot commands
291 CHROOT_LOC="`whereis -b chroot | cut -d " " -f2`"
292 i=1
293 for file in ../chroot-scripts/*chroot* ; do
294 chroot=`cat $file | \
295 perl -pe 's|\\\\\n||g' | \
296 tr -s [:space:] | \
297 grep chroot | \
298 sed -e "s|chroot|$CHROOT_LOC|" \
299 -e 's|\\$|&&|g' \
300 -e 's|"$$LFS"|$(MOUNT_PT)|'`
301 echo -e "CHROOT$i= $chroot\n" >> $MKFILE
302 i=`expr $i + 1`
303 done
304
305 # If CPUSET is defined and not equal to "all", then we define a first target
306 # that calls a script which re-enters make calling target all
307 if [ -n "$CPUSET" ] && [ "$CPUSET" != all ]; then
308(
309 cat << EOF
310
311all-with-cpuset:
312 @CPUSPEC="\$(CPUSET)" ./run-in-cgroup \$(MAKE) all
313EOF
314) >> $MKFILE
315 fi
316 # Drop in the main target 'all:' and the chapter targets with each sub-target
317 # as a dependency. Also prevent running targets in parallel.
318(
319 cat << EOF
320
321.NOTPARALLEL:
322
323all: ck_UID ck_terminal mk_SETUP mk_LUSER mk_SUDO mk_CHROOT mk_BOOT create-sbu_du-report mk_BLFS_TOOL mk_CUSTOM_TOOLS
324 @sudo env LFS=\$(MOUNT_PT) kernfs-scripts/teardown.sh
325EOF
326) >> $MKFILE
327# With systemd, the commands in chapter 9 do not create a
328# valid /etc/resolv.conf, so that if blfs_tools are installed, and
329# make-ca is run, it cannot connect. In this case, resolv.conf
330# from the host has been copied, and we need to remove it now.
331if [ "$INITSYS" = systemd ]; then
332(
333 cat << EOF
334 @sudo rm -v \$(MOUNT_PT)/etc/resolv.conf
335EOF
336) >> $MKFILE
337fi
338(
339 cat << EOF
340 @sudo make do_housekeeping
341 @echo $VERSION > lfs-release && \\
342 sudo mv lfs-release \$(MOUNT_PT)/etc && \\
343 sudo chown root:root \$(MOUNT_PT)/etc/lfs-release
344 @/bin/echo -e -n \\
345 DISTRIB_ID=\\"Linux From Scratch\\"\\\\n\\
346 DISTRIB_RELEASE=\\"$VERSION\\"\\\\n\\
347 DISTRIB_CODENAME=\\"$(whoami)-jhalfs\\"\\\\n\\
348 DISTRIB_DESCRIPTION=\\"Linux From Scratch\\"\\\\n\\
349 > lsb-release && \\
350 sudo mv lsb-release \$(MOUNT_PT)/etc && \\
351 sudo chown root:root \$(MOUNT_PT)/etc/lsb-release
352 @/bin/echo -e -n \\
353 NAME=\\"Linux From Scratch\\"\\\\n\\
354 VERSION=\\"$VERSION\\"\\\\n\\
355 ID=lfs\\\\n\\
356 PRETTY_NAME=\\"Linux From Scratch $VERSION\\"\\\\n\\
357 VERSION_CODENAME=\\"$(whoami)-jhalfs\\"\\\\n\\
358 > os-release && \\
359 sudo mv os-release \$(MOUNT_PT)/etc && \\
360 sudo chown root:root \$(MOUNT_PT)/etc/os-release
361 @\$(call echo_finished,$VERSION)
362
363ck_UID:
364 @if [ \`id -u\` = "0" ]; then \\
365 echo "--------------------------------------------------"; \\
366 echo "You cannot run this makefile from the root account"; \\
367 echo "--------------------------------------------------"; \\
368 exit 1; \\
369 fi
370
371ck_terminal:
372 @stty size | ( read L C; \\
373 if (( L < 24 )) || (( C < 80 )) ; then \\
374 echo "--------------------------------------------------"; \\
375 echo "Terminal too small: \$\$C columns x \$\$L lines";\\
376 echo "Minimum: 80 columns x 24 lines";\\
377 echo "--------------------------------------------------"; \\
378 exit 1; \\
379 fi )
380
381mk_SETUP:
382 @\$(call echo_SU_request)
383 @sudo make save-luser
384 @sudo make BREAKPOINT=\$(BREAKPOINT) SETUP
385 @touch \$@
386
387mk_LUSER: mk_SETUP
388 @\$(call echo_SULUSER_request)
389 @\$(SU_LUSER) "make -C \$(MOUNT_PT)/\$(SCRIPT_ROOT) BREAKPOINT=\$(BREAKPOINT) LUSER"
390 @sudo make restore-luser
391 @touch \$@
392
393mk_SUDO: mk_LUSER
394 @sudo rm -f envars
395 @sudo make BREAKPOINT=\$(BREAKPOINT) SUDO
396 @touch \$@
397
398mk_CHROOT: mk_SUDO devices
399 @\$(call echo_CHROOT_request)
400 @( sudo \$(CHROOT1) -c "cd \$(SCRIPT_ROOT) && make BREAKPOINT=\$(BREAKPOINT) CHROOT")
401 @touch \$@
402
403mk_BOOT: mk_CHROOT devices
404 @\$(call echo_CHROOT_request)
405 @( sudo \$(CHROOT1) -c "cd \$(SCRIPT_ROOT) && make BREAKPOINT=\$(BREAKPOINT) BOOT")
406 @touch \$@
407
408mk_BLFS_TOOL: create-sbu_du-report devices
409 @if [ "\$(ADD_BLFS_TOOLS)" = "y" ]; then \\
410 \$(call sh_echo_PHASE,Building BLFS_TOOL); \\
411 (sudo \$(CHROOT1) -c "make -C $BLFS_ROOT/work"); \\
412 fi;
413 @touch \$@
414
415mk_CUSTOM_TOOLS: mk_BLFS_TOOL devices
416 @if [ "\$(ADD_CUSTOM_TOOLS)" = "y" ]; then \\
417 \$(call sh_echo_PHASE,Building CUSTOM_TOOLS); \\
418 sudo mkdir -p ${BUILDDIR}${TRACKING_DIR}; \\
419 (sudo \$(CHROOT1) -c "cd \$(SCRIPT_ROOT) && make BREAKPOINT=\$(BREAKPOINT) CUSTOM_TOOLS"); \\
420 fi;
421 @touch \$@
422
423devices: ck_UID
424 sudo env LFS=\$(MOUNT_PT) kernfs-scripts/devices.sh
425EOF
426) >> $MKFILE
427# With systemd, the commands in chapter 9 do not create a
428# valid /etc/resolv.conf, so that if we want to enter chroot,
429# and systemd has not yet been run (otherwise /etc/resolv.conf
430# exists), we just copy resolv.conf from the host.
431if [ "$INITSYS" = systemd ]; then
432(
433 cat << EOF
434 if ! [ -e \$(MOUNT_PT)/etc/resolv.conf ]; then \\
435 sudo cp -v /etc/resolv.conf \$(MOUNT_PT)/etc; \\
436 sudo touch \$(MOUNT_PT)/etc/.host-resolvconf; \\
437 fi
438EOF
439) >> $MKFILE
440fi
441(
442 cat << EOF
443
444teardown:
445 sudo env LFS=\$(MOUNT_PT) kernfs-scripts/teardown.sh
446EOF
447) >> $MKFILE
448# With systemd, the commands in chapter 9 do not create a
449# valid /etc/resolv.conf, so that if we want to enter chroot,
450# and systemd has not yet been run (otherwise /etc/resolv.conf
451# exists), we just copy resolv.conf from the host.
452# We need to remove it now, if it has been created.
453if [ "$INITSYS" = systemd ]; then
454(
455 cat << EOF
456 if [ -e \$(MOUNT_PT)/etc/.host-resolvconf ]; then \\
457 sudo rm -v \$(MOUNT_PT)/etc/resolv.conf; \\
458 sudo rm -v \$(MOUNT_PT)/etc/.host-resolvconf; \\
459 fi
460EOF
461) >> $MKFILE
462fi
463(
464 cat << EOF
465
466chroot1: devices
467 -sudo \$(CHROOT1)
468 \$(MAKE) teardown
469
470chroot: devices
471 -sudo \$(CHROOT1)
472 \$(MAKE) teardown
473
474SETUP: $SETUP_TGT
475LUSER: $LUSER_TGT
476SUDO: $SUDO_TGT
477EOF
478) >> $MKFILE
479# With systemd, the commands in chapter 9 do not create a
480# valid /etc/resolv.conf, so that if blfs_tools are installed, and
481# make-ca is run, it cannot connect. We just copy resolv.conf
482# from the host. We'll remove it at the end.
483if [ "$INITSYS" = systemd ]; then
484(
485 cat << EOF
486 @cp -v /etc/resolv.conf \$(MOUNT_PT)/etc
487
488EOF
489) >> $MKFILE
490fi
491(
492 cat << EOF
493CHROOT: SHELL=\$(filter %bash,\$(CHROOT1))
494CHROOT: $CHROOT_TGT
495BOOT: $BOOT_TGT
496CUSTOM_TOOLS: $custom_list
497
498create-sbu_du-report: mk_BOOT
499 @\$(call echo_message, Building)
500 @if [ "\$(ADD_REPORT)" = "y" ]; then \\
501 sudo ./create-sbu_du-report.sh logs $VERSION $(date --iso-8601); \\
502 \$(call echo_report,$VERSION-SBU_DU-$(date --iso-8601).report); \\
503 fi
504 @touch \$@
505
506save-luser:
507 @\$(call echo_message, Building)
508 @LUSER_ID=\$\$(grep '^\$(LUSER):' /etc/passwd | cut -d: -f3); \\
509 if [ -n "\$\$LUSER_ID" ]; then \\
510 if [ ! -d \$(LUSER_HOME).XXX ]; then \\
511 mv \$(LUSER_HOME){,.XXX}; \\
512 mkdir \$(LUSER_HOME); \\
513 chown \$(LUSER):\$(LGROUP) \$(LUSER_HOME); \\
514 fi; \\
515 echo "\$\$LUSER_ID" > luser-id; \\
516 echo User \$(LUSER) exists with ID \$\$LUSER_ID; \\
517 else \\
518 rm -f luser-id; \\
519 echo User \$(LUSER) does not exist; \\
520 echo It will be created with book instructions.; \\
521 fi
522 @\$(call housekeeping)
523
524restore-luser:
525 @\$(call echo_message, Building)
526 @if [ -f luser-id ]; then \\
527 rm -rf \$(LUSER_HOME); \\
528 mv \$(LUSER_HOME){.XXX,}; \\
529 rm luser-id; \\
530 else \\
531 userdel \$(LUSER); \\
532 groupdel \$(LGROUP); \\
533 rm -rf \$(LUSER_HOME); \\
534 fi
535 @\$(call housekeeping)
536
537do_housekeeping:
538 @-rm -f /tools
539
540EOF
541) >> $MKFILE
542
543 # Bring over the items from the Makefile.tmp
544 cat $MKFILE.tmp >> $MKFILE
545 rm $MKFILE.tmp
546 echo "Creating Makefile... ${BOLD}DONE${OFF}"
547}
Note: See TracBrowser for help on using the repository browser.