From f2064ae3b61a5dc7f9701fec353ac4a73c209d25 Mon Sep 17 00:00:00 2001
From: David Lichteblau <david@lichteblau.com>
Date: Fri, 8 Jun 2012 21:22:39 +0200
Subject: [PATCH] make-target-contrib.sh: Build contribs in parallel using
 make -j if possible.

Grovel .asd files using Lisp to find out dependencies.

Use make to build and tests contribs in parallel.

Set SBCL_JOBS around make.sh to specify a manual value for -j.  By default,
we autodetect the number of CPUs on Linux and Darwin, or fall back to 1
otherwise.

The result of this work is slightly worse performance than the original
solution on single-CPU system, but an improvement of around 40% on a
quad-core CPU.
---
 contrib/asdf-stub.lisp          |    4 +-
 make-target-contrib-grovel.lisp |  112 +++++++++++++++++++++++++++++++++++++++
 make-target-contrib.sh          |   83 ++++++++++++++++++++---------
 3 files changed, 171 insertions(+), 28 deletions(-)
 create mode 100644 make-target-contrib-grovel.lisp

diff --git a/contrib/asdf-stub.lisp b/contrib/asdf-stub.lisp
index 1baef5f..bd99729 100644
--- a/contrib/asdf-stub.lisp
+++ b/contrib/asdf-stub.lisp
@@ -4,9 +4,7 @@
   (push :sb-building-contrib *features*)
   (asdf:operate 'asdf:load-op *system*)
   (let ((stub (make-pathname :name *system* :type "lisp")))
-    (when (probe-file (compile-file-pathname stub))
-      (error "fasl file exists"))
-    (with-open-file (s stub :direction :output :if-exists :error)
+    (with-open-file (s stub :direction :output :if-exists :supersede)
       (print '(unless (member "ASDF" *modules* :test #'string=)
                (require :asdf))
              s)
diff --git a/make-target-contrib-grovel.lisp b/make-target-contrib-grovel.lisp
new file mode 100644
index 0000000..3d0b240
--- /dev/null
+++ b/make-target-contrib-grovel.lisp
@@ -0,0 +1,112 @@
+;;;; make-target-contrib-grovel.lisp -- used by make-target-contrib-grovel.sh
+;;;;
+;;;; Grovel the dependencies of contribs and write a makefile capable of
+;;;; building them all in an appropriate order.
+
+;;;; First, define a dummy version of the ASDF package, which merely
+;;;; records dependencies.  We also need a dummy SB-GROVEL package, since
+;;;; SB-GROVEL, being an ASDF extension, is a system-definition-time
+;;;; dependency.
+
+(defpackage "DUMMY-ASDF"
+  (:use)
+  (:export "DEFSYSTEM" "FIND-SYSTEM" "LOAD-OP" "COMPILE-OP" "TEST-OP"
+           "PERFORM" "OOS" "OPERATE" "COMPONENT-PATHNAME" "OUTPUT-FILES")
+  (:nicknames "ASDF"))
+
+(defpackage "DUMMY-SB-GROVEL"
+  (:use)
+  (:export "GROVEL-CONSTANTS-FILE")
+  (:nicknames "SB-GROVEL"))
+
+(defpackage "DUMMY-ASDF-USER"
+  (:use "CL" "DUMMY-ASDF"))
+
+(defparameter *grovelled-dependencies* (make-hash-table :test 'equal))
+
+(defun record (system kind deps)
+  (setf (gethash (string-downcase system)
+                 *grovelled-dependencies*)
+        (cons kind deps)))
+
+(declaim (special *magic-deps*))
+
+(defmacro dummy-asdf:defsystem (name &key depends-on &allow-other-keys)
+  `(record ',name "LOADOP " (append ',depends-on *magic-deps* '("asdf"))))
+
+(defclass dummy-asdf:load-op () ())
+(defclass dummy-asdf:test-op () ())
+(defclass dummy-asdf:compile-op () ())
+
+(defun dummy-asdf:find-system (x) x)
+(defgeneric dummy-asdf:perform (operation component))
+(defgeneric dummy-asdf:output-files (x y))
+(defgeneric dummy-asdf:component-pathname (component))
+
+(defun dummy-asdf:operate (how what)
+  (declare (ignore how))
+  (push what *magic-deps*))
+
+(defun dummy-asdf:oos (how what)
+  (dummy-asdf:operate how what))
+
+;;;; With DUMMY-ASDF in place, load all .asds.  Contribs without an .asd
+;;;; file are "vanilla contribs" with a suitable makefile and no further
+;;;; dependencies.
+
+(let ((contribs (mapcar (lambda (dir)
+                          (list (car (last (pathname-directory dir)))
+                                dir))
+                        *contribs-to-build*)))
+  (dolist (cons contribs)
+    (destructuring-bind (basename dir) cons
+      (let ((asd (make-pathname :name basename :type "asd" :defaults dir)))
+        (if (probe-file asd)
+            (let ((*magic-deps* nil)
+                  (*package* (find-package "DUMMY-ASDF-USER"))
+                  (sb-ext:*module-provider-functions*
+                   (list (lambda (what)
+                           (dummy-asdf:operate :dummy what)))))
+              (load asd))
+            (record basename "VANILLA" nil)))))
+
+  ;; Generate .mak.
+
+  (with-open-file (s "make-target-contrib.mak"
+                     :direction :output
+                     :if-exists :supersede)
+    (format s "all: ~{~(~A~)-test~^ ~}~%~%"
+            (mapcar #'car contribs))
+    (flet ((rule (left right)
+             (format s "~(~A~): ~{~(~A~)~^ ~}~%"
+                     left right))
+           (cmd (fmt &rest args)
+             (format s "~C@~?~%" (code-char 9) fmt args))
+           (concat (sym suffix)
+             (format nil "~(~A~)-~A" sym suffix)))
+      (dolist (cons contribs)
+        (let ((sys (car cons)))
+          (destructuring-bind (kind &rest deps)
+              (or (gethash sys *grovelled-dependencies*)
+                  (error "failed to grovel dependencies for: ~A" sys))
+            ;;
+            ;; build rule
+            (rule (concat sys "build")
+                  (mapcar (lambda (x) (concat x "build")) deps))
+            (cmd "echo '  ~A' ~(~A~)" kind sys)
+            (cmd
+             "$(GNUMAKE) -C contrib/~(~A~) >output/building-contrib.~(~:*~A~) 2>&1"
+             sys)
+            (terpri s)
+
+            ;;
+            ;; test rule
+            (rule (concat sys "test") (list (concat sys "build")))
+            (cmd "echo '  TEST   ' ~(~A~)" sys)
+            (cmd
+             "$(GNUMAKE) -C contrib/~(~A~) test >>output/building-contrib.~(~:*~A~) 2>&1"
+             sys)
+            (cmd "touch contrib/~A/test-passed" sys)
+            (terpri s)
+            (terpri s)))))))
+(exit :code 0)
diff --git a/make-target-contrib.sh b/make-target-contrib.sh
index 22fe7e5..0b5b48f 100644
--- a/make-target-contrib.sh
+++ b/make-target-contrib.sh
@@ -69,38 +69,71 @@ else
     done
 fi
 
-for i in $contribs_to_build; do
-    test -d $i && test -f $i/Makefile || continue;
-    # export INSTALL_DIR=$SBCL_HOME/`basename $i `
-    test -f $i/test-passed && rm $i/test-passed
-    # hack to get exit codes right.
-    if $GNUMAKE -C $i test 2>&1 && touch $i/test-passed ; then
-	:
-    else
-	exit $?
-    fi | tee output/building-contrib.`basename $i` 
+unset processed_contribs_to_build
+for path in $contribs_to_build; do
+    test -d $path && test -f $path/Makefile || continue;
+    test -f $path/test-passed && rm $path/test-passed
+    rm -f output/$(basename $path)-build.stamp
+    rm -f output/$(basename $path)-test.stamp
+    processed_contribs_to_build="\"$path/\" $processed_contribs_to_build"
 done
 
-# Sometimes people used to see the "No tests failed." output from the last
-# DEFTEST in contrib self-tests and think that's all that is. So...
-HEADER_HAS_BEEN_PRINTED=false
+./src/runtime/sbcl \
+    --core output/sbcl.core \
+    --disable-debugger \
+    --lose-on-corruption \
+    --no-sysinit --no-userinit \
+    <<eof
+(defparameter *contribs-to-build* '($processed_contribs_to_build))
+(load "make-target-contrib-grovel.lisp")
+eof
+echo
+
+if [ -z SBCL_JOBS ]; then
+    case `uname` in
+        Linux)
+            SBCL_JOBS=$(grep -c ^processor /proc/cpuinfo)
+            ;;
+        Darwin)
+            SBCL_JOBS=$(system_profiler | awk '/Number of CPUs/ {print $$4}{next;}')
+            ;;
+        *)
+            SBCL_JOBS=1
+            ;;
+    esac
+fi
+
+make -j"$SBCL_JOBS" -w -f make-target-contrib.mak GNUMAKE="$GNUMAKE" || true
+
+unset failed_contribs
 for dir in contrib/*
 do
   if [ -d "$dir" -a -f "$dir/Makefile" -a ! -f "$dir/test-passed" ]; then
-      if $HEADER_HAS_BEEN_PRINTED; then
-          echo > /dev/null
-      else
-          cat <<EOF
+      failed_contribs="$failed_contribs `basename $dir`"
+  fi
+done
+
+# Traditionally, we printed all contrib building output directly while
+# also tee'ing to a file.  In a multiprocessing scenario, output would
+# interleave, so we save only to files with tee.
+#
+# We also have nicely compact output from make now.
+#
+# Remove comments from the following lines to see all contrib output in
+# the traditional fashion.
+#echo Recorded contrib build logs follow.
+#cat output/building-contrib* || true
+
+# Sometimes people used to see the "No tests failed." output from the last
+# DEFTEST in contrib self-tests and think that's all that is. So...
+if [ -n "$failed_contribs" ]; then
+    cat <<EOF
 
 WARNING! Some of the contrib modules did not build successfully or pass
 their self-tests. Failed contribs:"
 EOF
-          HEADER_HAS_BEEN_PRINTED=true
-      fi
-      echo "  `basename $dir`"
-  fi
-done
-
-if [ $HEADER_HAS_BEEN_PRINTED = true ]; then
-  exit 1
+    for basename in $failed_contribs; do echo "  $basename"; done
+    exit 1
+else
+    echo All contribs built.
 fi
-- 
1.7.9.5