source: BLFS/libs/func_dependencies@ e57e029

ablfs-more legacy trunk
Last change on this file since e57e029 was fd4a798, checked in by Pierre Labastie <pierre.labastie@…>, 3 years ago

Remove $Id$ comments, they are useless with git

  • Property mode set to 100644
File size: 19.9 KB
Line 
1#!/bin/bash
2#
3
4#-----------------------------------------------------------------------------#
5# This is a set of (recursive) functions for manipulating a dependency graph. #
6# We use algorithms and definitions from chapter 4 (mainly section 4.2) of #
7# https://algs4.cs.princeton.edu/. The graph we manipulate is the directed #
8# graph of the dependencies: nodes are packages in the BLFS book. A node A is #
9# connected to a node B if package A depends on B. A topological order (rev- #
10# erted) is exactly what we want for a build order. But a topological order #
11# only exists if the graph is acyclic. We'll therefore have to remove cycles. #
12# There are a number of other features we want to consider: #
13# - edges are weighted according to the dependency requirement: #
14# 1 for required #
15# 2 for recommended #
16# 3 for optional #
17# 4 for external #
18# We should consider only edges with weight lower or equal to that #
19# specified by the user, but see below. #
20# - we do not want to build the whole book. The user specifies a set of #
21# packages, and we have to consider only nodes reachable from this set #
22# using edges of weight not exceeding the specified weight. #
23# - when doing the topological sort, we want to consider all the edges and #
24# not only those not exceeding the specified weight: If a package A in the #
25# reachable subgraph depends optionally on another package B in the same #
26# subgraph, we want to build B before A. But this means we'll have to #
27# remove cycles for all weights. #
28# - dependencies have another qualifier: before or after. The problem: if a #
29# package A depends on B with an "after" qualifier, and a package C depends #
30# on A with a "before" qualifier, C may need B to be able to use A. So the #
31# only safe way to consider "after" qualifiers is to consider that they are #
32# "before" deps for any parent of the packages considered. #
33# We'll therefore have a 3 pass procedure. First build the set of nodes #
34# reachable from the root set. Second, remove dangling edges (those pointing #
35# to packages outside the node set), and move "after" edges to "before" edges #
36# originating from the parents. Third remove cycles and generate a #
37# topological sort. #
38# #
39# TODO: document each pass
40# Data layout: #
41# TODO: needs also to document the .tree files, and the "f" qualifier
42# #
43# A node of the tree is represented by a text file <nodeName>.dep. Each edge #
44# starting from this node is represented by a line in this file. We keep #
45# those files in the same directory. We introduce a special node named root, #
46# whose edges point to the list of nodes requested by the user. Each line #
47# contains three fields: #
48# - the weight of the edge #
49# - the "before" (b) or "after" (a) qualifier #
50# - the name of the destination of the edge #
51# #
52# TODO: The following is obsolete
53# Circular dependencies: #
54# #
55# In case we find a cirdular dependency, it has the form : #
56# parent->dependency_0->...->dependency_n->dependency_0 #
57# If we want to build dependency_n before dependency_0, no problem: #
58# we just prune the tree at dependency_n. If we want to build first #
59# dependency_0, we need to put dependency_n as a dependency of parent, #
60# then erase and rebuild the subtree from there. Now, we may have met #
61# another circular dependency in the subtree, and erasing the tree makes #
62# us forget the decision which was made. So, after first generating the #
63# list of dependencies from packages.xml, we keep the generated list in #
64# a file <nodeName>.odep, which we modify according to the decision which #
65# was made. #
66#---------------------------------------------------------------------------#
67
68# Global variables:
69# A string of spaces for indenting:
70declare -a spaceSTR=" "
71# When we are backing up from a circular dependency, `parentNode'
72# contains the node which has an edge entering the cycle
73declare parentNode
74
75#---------------------#
76generate_subgraph() { #
77#---------------------#
78: <<inline_doc
79 function: Create a subgraph of all the nodes reachable from the node
80 represented by the file whose name is $1. The edges considered
81 are those with maximal weight DEP_LEVEL (recursive function).
82 input vars: $1 : file name corresponding to the node whose edges will be
83 followed for the DFS
84 $2 : weight of the edge leading to this node
85 $3 : depth (root is 1)
86 $4 : qualifier (a for after, b for before)
87 externals: vars: DEP_LEVEL contains 1 if we want to build the
88 tree only for required dependencies,
89 2 if we want also recommended ones,
90 3 if we want also optional ones, but only
91 for the requested packages,
92 4 if we want all the dependencies
93 MAIL_SERVER contains the name of the MTA we want to use.
94 files: ../xsl/dependencies.xsl: stylesheet for creating the
95 .dep files
96 ../packages.xml: File containing packages id
97 and dependencies
98 returns: 0 if the tree has been successfully created
99 output: files: for each node reachable from $1, a file <node>.dep.
100 on error: nothing
101 on success: nothing
102inline_doc
103
104local depFile=$1
105local -i weight=$2
106local -i depth=$3
107local qualifier=$4
108local -i spacing=0
109local priostring
110local buildstring
111local id_of_dep
112local prio_of_dep
113local build_of_dep
114local dep_level
115
116if (( depth < 10 )); then spacing=1; fi
117case $weight in
118 1) priostring=required ;;
119 2) priostring=recommended ;;
120 3) priostring=optional ;;
121esac
122case $qualifier in
123 a) buildstring=runtime ;;
124 b) buildstring= ;;
125esac
126dep_level=$DEP_LEVEL
127if [ "$dep_level" = 3 ] && [ "$depth" -gt 2 ]; then dep_level=2; fi
128if [ "$dep_level" -gt 3 ]; then dep_level=3; fi
129echo -en "\nNode: $depth${spaceSTR:0:$(( depth + spacing ))}${RED}${depFile%.dep}${OFF} $priostring $buildstring"
130
131depth=$(( depth + 1 ))
132if (( depth < 10 )); then spacing=1; else spacing=0; fi
133# Start of loop
134{
135while read prio_of_dep build_of_dep id_of_dep; do
136 case $prio_of_dep in
137 1) priostring=required ;;
138 2) priostring=recommended ;;
139 3) priostring=optional ;;
140 4) priostring=external ;;
141 esac
142 case $build_of_dep in
143 a) buildstring=runtime ;;
144 b) buildstring= ;;
145 esac
146# Has this entry already been seen?
147# We have a special case here: if the entry has been seen at depth > 2
148# and now depth=2 and DEP_LEVEL=3, optional deps have not been processed.
149# If this is the case, just consider it has not been seen.
150 if [ -f ${id_of_dep}.dep ] ; then
151 case $depth$DEP_LEVEL in
152 23) ;;
153 *)
154# Just display it and proceed.
155 echo -en "\nEdge: $depth${spaceSTR:0:$((depth + spacing))}${MAGENTA}${id_of_dep}${OFF} $priostring $buildstring"
156 continue
157 ;;
158 esac
159 fi
160# Is the weight higher than requested?
161 if [ "$prio_of_dep" -gt $dep_level ]; then
162# Just display it and proceed.
163 echo -en "\n Out: $depth${spaceSTR:0:$((depth + spacing))}${YELLOW}${id_of_dep}${OFF} $priostring $buildstring"
164 continue
165 fi
166# Otherwise, let's build the corresponding subgraph.
167 xsltproc --stringparam idofdep "$id_of_dep" \
168 --stringparam MTA "$MAIL_SERVER" \
169 -o ${id_of_dep}.dep \
170 ../xsl/dependencies.xsl ../packages.xml
171
172 if [[ -s ${id_of_dep}.dep ]]; then # this dependency has dependencies
173 generate_subgraph ${id_of_dep}.dep $prio_of_dep $depth $build_of_dep
174 else # id_of_dep has no dependencies, just touch the file and display
175 touch ${id_of_dep}.dep
176 echo -en "\nLeaf: $depth${spaceSTR:0:$((depth + spacing))}${CYAN}${id_of_dep}${OFF} $priostring $buildstring"
177 fi
178done
179} <$depFile
180depth=$(( depth - 1 ))
181if (( depth < 10 )); then spacing=1; else spacing=0; fi
182echo -en "\n End: $depth${spaceSTR:0:$((depth + spacing))}${GREEN}${depFile%.dep}${OFF}"
183return 0
184}
185
186#-----------#
187path_to() { #
188#-----------#
189: <<inline_doc
190 function: check whether there is a path from $1 to $2 on the graph
191 input vars: $1 contains the filename of the starting node.
192 $2 contains the name of the node to reach
193 $3 contains the weight above which we do not want to
194 follow an edge
195 "seen" (global) contains the list of already seen nodes
196 returns: 0 if the node has been found
197 1 if not
198 on error: nothing
199 on success: nothing
200inline_doc
201local start=$1
202local seek=$2
203local prio=$3
204local prio_of_dep
205local build_of_dep
206local id_of_dep
207local r
208
209if test "${start%.dep}" = "$seek"; then return 0; fi
210seen="$seen${start%.dep} "
211if test -s $start; then
212 {
213 while read prio_of_dep build_of_dep id_of_dep; do
214 if test "$prio" -lt "$prio_of_dep"; then continue; fi
215 if ! test "${seen% $id_of_dep *}" = "$seen"; then continue; fi
216 if path_to ${id_of_dep}.dep $seek $prio; then return 0; fi
217 done
218 } < $start
219fi
220return 1
221}
222#------------------#
223clean_subgraph() { #
224#------------------#
225: <<inline_doc
226 function: Remove dangling edges and create groups of deps for "after"
227 deps: A-before->B-after->C becomes:
228 A -before-> Bgroupxx -before-> B
229 \
230 -before-> C
231 the name of the group is chosen so that it is unlikely as
232 a package name (so that it is removed when building the
233 xml book).
234 input vars: None
235 files: <node>.dep files containing dangling edges and
236 "after" qualifiers
237 returns: 0
238 output: files: <node>.dep files containing no dangling edges and
239 no "after" qualifiers
240 on error: nothing
241 on success: nothing
242inline_doc
243
244local node
245local id_of_dep
246local prio_of_dep
247local build_of_dep
248local lines_to_remove
249local lines_to_change
250local parent
251local p
252local b
253local start
254local seen
255
256for node in $(ls *.dep); do
257 if test $node = root.dep; then continue; fi
258 echo Cleaning $node
259 lines_to_remove=
260 {
261 while read prio_of_dep build_of_dep id_of_dep; do
262 if ! test -f ${id_of_dep}.dep; then
263 lines_to_remove="$lines_to_remove $id_of_dep"
264 continue
265 fi
266 done
267 } <$node
268 for id_of_dep in $lines_to_remove; do
269 sed "/\ $id_of_dep\$/d" -i $node
270 done
271done
272for node in $(grep -l ' a ' *.dep); do
273 lines_to_remove=
274 echo Process "runtime" deps in $node
275 if ! [ -e ${node%.dep}groupxx.dep ]; then
276 b=0
277 for parent in $(grep -l ${node%.dep}\$ *); do
278 p=0
279 for start in $(grep ' a ' $node | cut -d' ' -f3); do
280 seen=" "
281 if path_to ${start}.dep ${parent%.dep} 3; then p=1; break; fi
282 done
283 if test $p = 0; then
284 b=1
285 sed -i "s/\ ${node%.dep}\$/&groupxx/" $parent
286 fi
287 done
288 echo "1 b ${node%.dep}" > ${node%.dep}groupxx.dep
289 if test $b = 0; then echo "1 b ${node%.dep}groupxx" >> root.dep; fi
290 fi
291 {
292 while read prio_of_dep build_of_dep id_of_dep; do
293 if test $build_of_dep = a; then
294 if ! grep -q ${id_of_dep} ${node%.dep}groupxx.dep; then
295 echo "$prio_of_dep b ${id_of_dep}" >> ${node%.dep}groupxx.dep
296 fi
297 lines_to_remove="$lines_to_remove $id_of_dep"
298 fi
299 done
300 } <$node
301 for id_of_dep in $lines_to_remove; do
302 sed "/\ $id_of_dep\$/d" -i $node
303 done
304done
305for node in $(grep -l ' f ' *); do
306 echo Process "first" deps in $node
307 lines_to_change=
308 {
309 while read prio_of_dep build_of_dep id_of_dep; do
310 if test $build_of_dep = f; then
311 if ! test -f ${id_of_dep}-pass1.dep; then
312 cp ${id_of_dep}{,-pass1}.dep;
313 fi
314 lines_to_change="$lines_to_change $id_of_dep"
315 unset lr
316 {
317 while read p b start; do
318 seen=" "
319 if path_to ${start}.dep ${node%.dep} $p; then
320 lr="$lr $start"
321 fi
322 done
323 } < ${id_of_dep}-pass1.dep
324 for p in $lr; do
325 sed "/\ $p\$/d" -i ${id_of_dep}-pass1.dep
326 done
327 fi
328 done
329 } <$node
330 for id_of_dep in $lines_to_change; do
331 sed "/\ $id_of_dep\$/"'{s/[[:digit:]] f /1 b /;s/$/-pass1/}' -i $node
332 done
333done
334} # End clean_subgraph
335
336#----------------------------#
337generate_dependency_tree() { #
338#----------------------------#
339: <<inline_doc
340 function: Create a subtree of the dependency tree
341 (recursive function)
342 input vars: $1 : file with a list of targets (child nodes)
343 the first line of the file is the link
344 $2 : priority (1=req, 2=rec, 3=opt)
345 returns: 0 if the tree has been successfully created
346 1 if we are backing up to the parent of a circular dep
347 and there are only required deps in the cycle
348 2 if we are backing up to the parent of a circular dep
349 and there are recommended deps and no optional deps in the
350 cycle
351 3 if we are backing up to the parent of a circular dep
352 and there are optiional deps in the cycle
353 modifies: vars: ParentNode is set when return is not 0
354 output: files: for each <pkg> with dependencies in $1,
355 a file <pkg>.tree and its dependencies
356 on error: nothing
357 on success: nothing
358inline_doc
359
360local depFile=$1
361local priority=$2
362local -a rootlink
363local -a priolink
364local -a otherlink
365local -i depth
366local -i count=0
367local id_of_dep
368local build_of_dep
369local prio_of_dep
370local parent
371local lines_to_remove=
372local srootlink
373local priostring
374local dpriostring
375local i
376
377{
378read -a rootlink
379depth=${#rootlink[*]}
380read -a priolink
381srootlink="${rootlink[*]} "
382case $priority in
383 1) priostring=required ;;
384 2) priostring=recommended ;;
385 3) priostring=optional ;;
386esac
387# start of depFile
388echo -en "\nNode: $depth${spaceSTR:0:$depth}${RED}${depFile%.tree}${OFF} $priostring"
389
390while read prio_of_dep build_of_dep id_of_dep; do
391 case $prio_of_dep in
392 1) dpriostring=required ;;
393 2) dpriostring=recommended ;;
394 3) dpriostring=optional ;;
395 esac
396# count entries in file
397 (( count++ ))
398# Has this entry already been seen?
399 if [ -f ${id_of_dep}.tree ]; then # found ${id_of_dep}.tree already in tree
400 otherlink=($(head -n 1 ${id_of_dep}.tree))
401 if [ -z "${otherlink[*]}" ]
402 then echo otherlink empty for $id_of_dep.tree
403 echo This should not happen, but happens to happen...
404 exit 1
405 fi
406#Do not use "${rootlink[*]}" =~ "${otherlink[*]}": case rootlink=(1 11)
407# and otherlink=(1 1)
408 if [[ ${srootlink#"${otherlink[*]} "} != ${srootlink} ]]; then # cir. dep
409 echo -en "\nCirc: $((depth+1))${spaceSTR:0:$((depth+1))}${YELLOW}${id_of_dep}${OFF} $dpriostring"
410# Find lowest priority in branch from parent to depFile:
411 p2=0
412 for (( i=${#otherlink[*]}; i < $depth ; i++ )) ; do
413 if (( ${priolink[i]} > $p2 )); then p2=${priolink[i]}; fi
414 done
415 if (( $prio_of_dep >= $p2 )); then # prune
416 lines_to_remove="$lines_to_remove $id_of_dep"
417 sed -i "/$id_of_dep/d" ${depFile/.tree/.dep}
418 else # find and set parent, then return lowest priority
419# The parent has the same link without the last entry.
420# We do not need otherlink anymore so just destroy the last element
421 unset otherlink[-1]
422 parentNode=$(grep ^"${otherlink[*]}"\$ -l *)
423 return $p2
424 fi
425 else # not circular: prune tree (but not .dep, since it may happen that
426 # the tree is destroyed and rebuilt in another order)
427 lines_to_remove="$lines_to_remove $id_of_dep"
428 fi # circular or not
429 continue # this dependency has already been seen, and the associated
430 # subtree computed. We are done
431 fi # Has this entry already been seen?
432# So, this entry has not already been seen. Let's build the corresponding
433# subtree. First check there is a subtree...
434# Use -s, because it may happen that after removing lines, .dep exists
435# but is empty.
436 if [[ -s ${id_of_dep}.dep ]]; then # this dependency has dependencies
437 sed "1i${rootlink[*]} $count\\
438${priolink[*]} $prio_of_dep" < ${id_of_dep}.dep \
439 > ${id_of_dep}.tree # add link and priolink
440 generate_dependency_tree ${id_of_dep}.tree $prio_of_dep
441# Test return value, in case we exchange dependencies
442 p2=$?
443 case $p2 in
444 0) # Normal return
445 ;;
446 $prio_of_dep) # we remove this dep, but since it may become unreachable,
447 # move it to be built later (as a dep of parent).
448 tree_erase ${id_of_dep}.tree
449 lines_to_remove="$lines_to_remove $id_of_dep"
450 sed -i "/${id_of_dep}/d" ${depFile/.tree/.dep}
451 echo "$prio_of_dep b $id_of_dep" >> $parentNode
452# must be added to .dep in case parentNode is destroyed when erasing
453# the tree
454 echo "$prio_of_dep b $id_of_dep" >> ${parentNode/.tree/.dep}
455 continue
456 ;;
457 *) # We are backing up
458 return $p2
459 ;;
460 esac
461 else # id_of_dep has no dependencies, just record the link in a file
462 # and print
463 echo "${rootlink[*]} $count" > ${id_of_dep}.tree
464 echo -en "\nLeaf: $(($depth+1))${spaceSTR:0:$(($depth+1))}${CYAN}${id_of_dep}${OFF} $dpriostring"
465 fi
466done
467echo -en "\n End: $depth${spaceSTR:0:$depth}${GREEN}${depFile%.tree}${OFF}"
468} <$depFile
469# It may happen that a file is created with several times
470# the same line. Normally, all those lines but one
471# would be flagged to be removed (or all of them if
472# the dependency appeared before). A simple sed /$line/d
473# destroys all the lines. We should instead remove
474# only one for each appearance of it in lines_to_remove.
475# so first get the position of last line and then delete
476# that line
477for line in $lines_to_remove
478 do lineno=$(sed -n /^[[:digit:]]\ b\ $line\$/= $depFile | tail -n1)
479 sed -i ${lineno}d $depFile
480done
481return 0
482}
483
484
485#---------------#
486tree_browse() { #
487#---------------#
488local file=$1
489local f
490
491#echo file=$file
492for f in $(grep '[^0-9 ]' $file | sed 's/.* //'); do
493# echo f=$f
494 if grep -q '[^0-9 ]' ${f}.tree ; then
495 tree_browse ${f}.tree
496 fi
497 echo $f
498done
499}
500
501#--------------#
502tree_erase() { #
503#--------------#
504local file=$1
505local f
506local rootlink
507local rootlink2
508
509#echo file=$file
510rootlink="$(head -n1 $file) "
511for f in $(grep '[^0-9 ]' $file | sed 's/.* //'); do
512 if [ -f ${f}.tree ]; then
513 rootlink2="$(head -n1 ${f}.tree) "
514# We want two things:
515# i) do not erase the file if it is in another branch
516# ii) do not erase the file if there is a circular dependency
517# for case i), we test that rootlink is contained in rootlink2
518# for case ii), we test that rootlink2 is not contained in
519# rootlink.
520# See comment above about srootlink
521 if [[ ${rootlink2#${rootlink}} != ${rootlink2} &&
522 ${rootlink#${rootlink2}} == ${rootlink} ]] ; then
523 tree_erase ${f}.tree
524 fi
525 fi
526done
527rm -f $file
528}
Note: See TracBrowser for help on using the repository browser.