Changes in / [bc64795:9440e4a]


Ignore:
Files:
1 added
10 edited

Legend:

Unmodified
Added
Removed

  • Config.in

    rbc64795 r9440e4a  
     1# Check for "nproc" presence:
     2config HAVE_NPROC
     3    def_bool $(shell,if nproc &>/dev/null; then echo y; else echo n; fi)
     4
     5# Check for "cpuset controller in cgroup v2" presence:
     6config HAVE_CGROUP
     7    def_bool $(shell,if grep -q cpuset /sys/fs/cgroup/cgroup.controllers 2>/dev/null; then echo y; else echo n; fi)
     8
    19menu "BOOK Settings"
    210
    … …  
    751759depends on !BOOK_BLFS
    752760
     761    #--- Parallelism
     762    menu "Parallelism settings"
     763    if HAVE_NPROC
     764        config ALL_CORES
     765        bool "Use all cores as in new books (say n for old books)"
     766        default y
     767        help
     768              Use book instructions as written for parallelism since version
     769              r12.0-87 included. If you answer no, then jhalfs will fall back
     770              to a static number of cores, defined below. If you answer y, and
     771              your system supports it, you'll be asked for a cpu set to use.
     772                 There is no detection of book version. If the book version
     773              is lower than r12.0-87, say no!
     774
     775        if ALL_CORES && HAVE_CGROUP
     776            config CPUSET
     777            string "set of cpus to use, or 'all' for all cpus"
     778            default "all"
     779            help
     780                 See "List format" in cpuset(7). Choosing cpus depend
     781                 on the topology of your processors. Sometimes two
     782                 hyperthreads on the same core are numbered consecutively.
     783                 For example for using all cores and no hyperthreading on
     784                 a Haswell, use "0,2,4,6". Other brands may have a different
     785                 topology, and may require e.g. "0-3" to use the first 4 cores.
     786                 If not sure, keep the default.
     787        endif
     788
     789    endif # HAVE_NPROC
     790    if !HAVE_NPROC
     791        config ALL_CORES
     792        bool
     793        default n
     794    endif
     795
     796    config    N_PARALLEL
     797        int "Number of parallel `make' jobs"
     798        depends on !ALL_CORES
     799        default 1
     800        help
     801            #-- The usual recommandation is (number of CPU cores)+1
     802            #   Do not set for meaningful SBU calculations.
     803
     804    config    REALSBU
     805        bool    "Build Binutils pass1 without parallelism (Real SBU)"
     806        default    n
     807        help
     808           #-- Use -j1 in make invokation for Binutils pass1 to
     809           #   get a valid SBU value.
     810    endmenu # parallelism
     811
     812    #--- Optimizations
     813    config    CONFIG_OPTIMIZE
     814        bool    "Optimization"
     815        default    n
     816        help
     817            #   Opens a menu for various optimization settings:
     818            #   Actual optimization flags MUST be defined in ./optimize/*
     819            #   before activating this option.
     820            #
     821            # WARNING: The use of build optimizations may lead to build issues.
     822            #   If the system doesn't work as expected, please rebuild
     823            #   without optimizations before asking for support.
     824    menu    "Optimization settings"
     825        depends on CONFIG_OPTIMIZE
     826
     827    choice
     828        prompt "Optimization level"
     829        default OPT_1
     830        help
     831            #-- Optimization values are set in optimize/* files
     832
     833        config    OPT_1
     834            bool    "Final system only"
     835
     836        config    OPT_2
     837            bool    "Both temp tools and final system"
     838
     839    endchoice
     840
     841    endmenu # Optimization settings
     842    config    OPTIMIZE
     843        int
     844        default    "0"    if !CONFIG_OPTIMIZE
     845        default    "1"    if OPT_1
     846        default    "2"    if OPT_2
     847
     848    #--- End Optimizations
     849
    753850    config    REPORT
    754851        bool "Create SBU and disk usage report"
    … …  
    796893
    797894    #--- End ICA
    798 
    799     #--- Optimizations
    800     config    CONFIG_OPTIMIZE
    801         bool    "Optimization and parallelization"
    802         default    n
    803         help
    804             #   Opens a menu for various optimization settings:
    805             #   Actual optimization flags MUST be defined in ./optimize/*
    806             #   before activating this option.
    807             #
    808             # WARNING: The use of build optimizations may lead to build issues.
    809             #   If the system doesn't work as expected, please rebuild
    810             #   without optimizations before asking for support.
    811     menu    "Parallelization and Optimization settings"
    812         depends on CONFIG_OPTIMIZE
    813 
    814     config    N_PARALLEL
    815         int "Number of parallel `make' jobs"
    816         default 1
    817         help
    818             #-- The usual recommandation is (number of CPU cores)+1
    819             #   Do not set for meaningful SBU calculations.
    820 
    821     choice
    822         prompt "Optimization level"
    823         default OPT_1
    824         help
    825             #-- Optimization values are set in optimize/* files
    826 
    827         config    OPT_1
    828             bool    "Final system only"
    829 
    830         config    OPT_2
    831             bool    "Both temp tools and final system"
    832 
    833     endchoice
    834 
    835     config    REALSBU
    836         bool    "Build Binutls pass1 without optimization (Real SBU)"
    837         depends on OPT_2
    838         default    n
    839         help
    840            #-- Use -j1 in make invokation for Binutils pass1 to
    841            #   get a valid SBU value.
    842 
    843     endmenu # Optimization settings
    844     config    OPTIMIZE
    845         int
    846         default    "0"    if !CONFIG_OPTIMIZE
    847         default    "1"    if OPT_1
    848         default    "2"    if OPT_2
    849 
    850     #--- End Optimizations
    851895
    852896    #-- Internal Settings

  • LFS/lfs.xsl

    rbc64795 r9440e4a  
    2828  -->
    2929  <xsl:param name="testsuite" select="1"/>
     30
     31  <!-- Parallelism (LFS >= 12.1) -->
     32  <xsl:param name="jobs" select="1"/>
     33
     34  <!-- value of jobs for binutils-pass1 -->
     35  <xsl:param name="jobs-bp1" select="1"/>
    3036
    3137  <!-- Install non wide character ncurses 5? -->
    … …  
    618624        <xsl:value-of select="$hostname"/>
    619625      </xsl:when>
     626      <xsl:when test="contains(string(.),'$(nproc)')">
     627        <xsl:value-of select="$jobs"/>
     628      </xsl:when>
    620629      <xsl:otherwise>
    621630        <xsl:text>**EDITME</xsl:text>
    … …  
    12441253</xsl:text>
    12451254    </xsl:if>
     1255    <xsl:text>
     1256export MAKEFLAGS="-j</xsl:text>
     1257    <xsl:choose>
     1258      <xsl:when test="@id='ch-tools-binutils-pass1'">
     1259        <xsl:value-of select="$jobs-bp1"/>
     1260      </xsl:when>
     1261      <xsl:otherwise>
     1262        <xsl:value-of select="$jobs"/>
     1263      </xsl:otherwise>
     1264    </xsl:choose>
     1265    <xsl:text>"
     1266</xsl:text>
    12461267    <xsl:text>SECONDS=${PREV_SEC}
    12471268

  • LFS/master.sh

    rbc64795 r9440e4a  
    128128      # If using optimizations, write the instructions
    129129      case "${OPTIMIZE}$1${nb_chaps}${this_script}${REALSBU}" in
    130           0* | *binutils-pass1y | 15* | 167* | 177*)
    131               wrt_makeflags "$name" "-j1" "1" ;;
    132           *kernel*) # No CFLAGS for kernel
    133               wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL" ;;
    134           *)  wrt_optimize "$name" &&
    135               wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL" ;;
     130          0* | *binutils-pass1y | 15* | 167* | 177*) ;;
     131          *kernel*) ;; # No CFLAGS for kernel
     132          *)  wrt_optimize "$name" ;;
    136133      esac
    137     fi
     134      # There is no need to tweak MAKEFLAGS anymore, this is done
     135      # by lfs.xsl. But still, NINJAJOBS needs to be set if
     136      # N_PARALLEL is defined.
     137      if [ -n "N_PARALLEL" ]; then
     138         wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL"
     139      fi
     140    fi # end of package specific instructions
    138141
    139142# Some scriptlet have a special treatment; otherwise standard
    … …  
    287290  done
    288291
     292  # If CPUSET is defined and not equal to "all", then we define a first target
     293  # that calls a script which re-enters make calling target all
     294  if [ -n "$CPUSET" ] && [ "$CPUSET" != all ]; then
     295(
     296    cat << EOF
     297
     298all-with-cpuset:
     299        @CPUSPEC="\$(CPUSET)" ./run-in-cgroup.sh \$(MAKE) all
     300EOF
     301) >> $MKFILE
     302  fi
    289303  # Drop in the main target 'all:' and the chapter targets with each sub-target
    290304  # as a dependency. Also prevent running targets in parallel.

  • common/chroot.xsl

    rbc64795 r9440e4a  
    66      extension-element-prefixes="exsl"
    77      version="1.0">
     8
     9  <xsl:param name="jobs_2" select="1"/>
    810
    911  <xsl:template match="/">
    … …  
    5658    <xsl:param name="instructions" select="''"/>
    5759    <xsl:choose>
     60      <xsl:when test="contains($instructions, '$(nproc || echo 1)')">
     61        <xsl:call-template name="extract-chroot">
     62          <xsl:with-param
     63            name="instructions"
     64            select="concat(substring-before($instructions, '$(nproc || echo 1)'), $jobs_2, substring-after($instructions, '$(nproc || echo 1)'))"/>
     65        </xsl:call-template>
     66      </xsl:when>
    5867      <xsl:when test="not(starts-with($instructions,'&#xA;chroot')) and
    5968                      contains($instructions, '&#xA;chroot')">

  • common/libs/func_book_parser

    rbc64795 r9440e4a  
    6464  popd > /dev/null
    6565
     66  if [ "$ALL_CORES" = "y" ]; then
     67          JOBS="\$(nproc)"
     68  else
     69          JOBS="$N_PARALLEL"
     70  fi
     71  if [ "$REALSBU" = y ]; then JOBSBP1=1; else JOBSBP1="$JOBS"; fi
     72
    6673  # First profile the book, for revision and arch. Note that
    6774  # MULTIBLIB is set to "default" if pure 64 bit book. In this case,
    … …  
    103110           --stringparam log-level      "$LOG_LEVEL"    \
    104111           --stringparam script-root    "$SCRIPT_ROOT"  \
     112           --stringparam jobs           "$JOBS"         \
     113           --stringparam jobs-bp1       "$JOBSBP1"      \
    105114           --output "./${COMMANDS}/"                    \
    106115           $XSL                                         \
    … …  
    142151  echo -n "... "
    143152  xsltproc --nonet --xinclude \
    144            -o chroot-scripts/ chroot.xsl \
     153           --stringparam jobs_2 "$JOBS_2" \
     154           -o chroot-scripts/ chroot.xsl  \
    145155           $BOOK/chapter0?/*chroot*.xml >> $LOGDIR/$LOG 2>&1
    146156  echo "done"

  • common/libs/func_install_blfs

    rbc64795 r9440e4a  
    137137BUILD_ROOT=/sources
    138138BUILD_SUBDIRS=y
    139 JOBS=$(if [ -n "$N_PARALLEL" ]; then echo $N_PARALLEL; else echo 1; fi)
     139JOBS=$(if [ "$ALL_CORES" = y ]; then echo 0; else echo $N_PARALLEL; fi)
    140140EOF
     141# The 0 value above is for using all cores
    141142for OPT_VAR in CFLAGS CXXFLAGS LDFLAGS; do
    142143  eval optVal=\$${OPT_VAR}_$DEF_OPT_MODE
    … …  
    218219# updating to the end of the process, adding an 'update' target
    219220sed -i -e '/xsltproc/,+6d' \
    220        -e '/^all/s@$@ update@' \
     221       -e '/^all/i update:' \
    221222       -e 's/touch/@touch/' Makefile
    222223cat >> Makefile << EOF
    223 update:
     224update: all
    224225        @echo Updating the tracking file
    225226        @for pack in \$\$(grep '<productname' ../$LFS_XML/tmp/lfs-full.xml | \\

  • common/libs/func_validate_configs.sh

    rbc64795 r9440e4a  
    3636
    3737  # Full list of books settings
    38   local -r   lfs_PARAM_LIST="$LFS_book   $GENERAL_common $LFS_build $LFS_system  $ADVANCED_chroot N_PARALLEL REALSBU SAVE_CH5 $ADVANCED_common"
     38  local -r   lfs_PARAM_LIST="$LFS_book   $GENERAL_common $LFS_build $LFS_system  $ADVANCED_chroot ALL_CORES CPUSET N_PARALLEL REALSBU SAVE_CH5 $ADVANCED_common"
    3939#  local -r  blfs_PARAM_LIST="BRANCH_ID BLFS_ROOT BLFS_XML TRACKING_DIR"
    4040
    … …  
    133133      RETRYDOWNLOADCNT) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;;
    134134      DOWNLOADTIMEOUT)  [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;;
    135       N_PARALLEL)       [[ "$OPTIMIZE" -gt "0" ]] && echo -e "`eval echo $PARAM_VALS`" ;;
    136       REALSBU)          [[ "$OPTIMIZE" = "2" ]] && echo -e "`eval echo $PARAM_VALS`" ;;
     135      CPUSET)           [[ "$HAVE_CGROUP" = "y" ]] && [[ "$ALL_CORES" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;;
     136      N_PARALLEL)       [[ "$ALL_CORES" = "n" ]] && echo -e "`eval echo $PARAM_VALS`" ;;
    137137
    138138      # Envars that requires some validation
    … …  
    167167
    168168        # Treatment of LANG parameter
    169       LANG )  # See it the locale value has been set
     169      LANG )  # See if the locale value has been set
    170170               echo -n "`eval echo $PARAM_VALS`"
    171171               [[ -z "${!config_param}" ]] &&

  • common/libs/func_wrt_Makefile

    rbc64795 r9440e4a  
    4545PKGMNGT          = $PKGMNGT
    4646WRAP_INSTALL     = $WRAP_INSTALL
     47CPUSET           = $CPUSET
    4748
    4849

  • jhalfs

    rbc64795 r9440e4a  
    176176UNICODE=${UNICODE:=n}
    177177LOCAL=${LOCAL:=n}
     178ALL_CORES=${ALL_CORES:=n}
    178179REALSBU=${REALSBU:=n}
    179180SAVE_CH5=${SAVE_CH5:=n}
    … …  
    288289  # optimize configurations
    289290  load_file optimize/opt_config 'Loading optimization config'
    290   # The number of parallel jobs is taken from configuration now
     291  # Validate optimize settings, if required
     292  validate_opt_settings
     293fi
     294# Parallelization is outside optimization, because it is now in the book
     295if [[ "$ALL_CORES" = "n" ]]; then
    291296  # shellcheck disable=SC2034
    292297  JH_MAKEFLAGS="-j${N_PARALLEL}"
    293   # Validate optimize settings, if required
    294   validate_opt_settings
    295298fi
    296299#
    … …  
    321324
    322325# Copy common helper files
    323   cp "$COMMON_DIR"/{makefile-functions,progress_bar.sh} "$JHALFSDIR/"
     326  cp "$COMMON_DIR"/{makefile-functions,progress_bar.sh,run-in-cgroup.sh} "$JHALFSDIR/"
    324327
    325328# Copy needed stylesheets

  • optimize/optimize_functions

    rbc64795 r9440e4a  
    1717  echo -e "asking for support.${OFF}\n"
    1818
    19   echo -e "MAKEFLAGS: ${L_arrow}${BOLD}${JH_MAKEFLAGS}${OFF}${R_arrow}"
    20   [ -n "$JH_MAKEFLAGS" ] && \
    21   echo -e "BLACK_LIST: ${L_arrow}${BOLD}${BLACK_LIST}${OFF}${R_arrow}\n"
     19#  [ -n "$JH_MAKEFLAGS" ] && \
     20#  echo -e "BLACK_LIST: ${L_arrow}${BOLD}${BLACK_LIST}${OFF}${R_arrow}\n"
    2221
    2322  echo -e "DEF_OPT_MODE: ${L_arrow}${BOLD}${DEF_OPT_MODE}${OFF}${R_arrow}\n"
Note: See TracChangeset for help on using the changeset viewer.