aboutsummaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorKees Cook <keescook@chromium.org>2019-11-21 12:59:29 -0800
committerJonathan Corbet <corbet@lwn.net>2019-11-22 10:35:18 -0700
commit51e46c7a4007d271b2d42dbc2df953ab968577a7 (patch)
tree9080a226fd6723c2c5ba3035ddf0034a3c8581dc /scripts
parentdocs, parallelism: Do not leak blocking mode to other readers (diff)
downloadlinux-dev-51e46c7a4007d271b2d42dbc2df953ab968577a7.tar.xz
linux-dev-51e46c7a4007d271b2d42dbc2df953ab968577a7.zip
docs, parallelism: Rearrange how jobserver reservations are made
Rasmus correctly observed that the existing jobserver reservation only worked if no other build targets were specified. The correct approach is to hold the jobserver slots until sphinx has finished. To fix this, the following changes are made: - refactor (and rename) scripts/jobserver-exec to set an environment variable for the maximally reserved jobserver slots and exec a child, to release the slots on exit. - create Documentation/scripts/parallel-wrapper.sh which examines both $PARALLELISM and the detected "-jauto" logic from Documentation/Makefile to decide sphinx's final -j argument. - chain these together in Documentation/Makefile Suggested-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Link: https://lore.kernel.org/lkml/eb25959a-9ec4-3530-2031-d9d716b40b20@rasmusvillemoes.dk Signed-off-by: Kees Cook <keescook@chromium.org> Link: https://lore.kernel.org/r/20191121205929.40371-4-keescook@chromium.org Signed-off-by: Jonathan Corbet <corbet@lwn.net>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/jobserver-count59
-rwxr-xr-xscripts/jobserver-exec66
2 files changed, 66 insertions, 59 deletions
diff --git a/scripts/jobserver-count b/scripts/jobserver-count
deleted file mode 100755
index 7807bfa7dafa..000000000000
--- a/scripts/jobserver-count
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python
-# SPDX-License-Identifier: GPL-2.0+
-#
-# This determines how many parallel tasks "make" is expecting, as it is
-# not exposed via an special variables.
-# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
-from __future__ import print_function
-import os, sys, fcntl, errno
-
-# Default parallelism is "1" unless overridden on the command-line.
-default="1"
-if len(sys.argv) > 1:
- default=sys.argv[1]
-
-# Extract and prepare jobserver file descriptors from envirnoment.
-try:
- # Fetch the make environment options.
- flags = os.environ['MAKEFLAGS']
-
- # Look for "--jobserver=R,W"
- # Note that GNU Make has used --jobserver-fds and --jobserver-auth
- # so this handles all of them.
- opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
-
- # Parse out R,W file descriptor numbers and set them nonblocking.
- fds = opts[0].split("=", 1)[1]
- reader, writer = [int(x) for x in fds.split(",", 1)]
- # Open a private copy of reader to avoid setting nonblocking
- # on an unexpecting process with the same reader fd.
- reader = os.open("/proc/self/fd/%d" % (reader),
- os.O_RDONLY | os.O_NONBLOCK)
-except (KeyError, IndexError, ValueError, IOError, OSError) as e:
- print(e, file=sys.stderr)
- # Any missing environment strings or bad fds should result in just
- # using the default specified parallelism.
- print(default)
- sys.exit(0)
-
-# Read out as many jobserver slots as possible.
-jobs = b""
-while True:
- try:
- slot = os.read(reader, 1)
- jobs += slot
- except (OSError, IOError) as e:
- if e.errno == errno.EWOULDBLOCK:
- # Stop when reach the end of the jobserver queue.
- break
- raise e
-# Return all the reserved slots.
-os.write(writer, jobs)
-
-# If the jobserver was (impossibly) full or communication failed, use default.
-if len(jobs) < 1:
- print(default)
- sys.exit(0)
-
-# Report available slots (with a bump for our caller's reserveration).
-print(len(jobs) + 1)
diff --git a/scripts/jobserver-exec b/scripts/jobserver-exec
new file mode 100755
index 000000000000..0fdb31a790a8
--- /dev/null
+++ b/scripts/jobserver-exec
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: GPL-2.0+
+#
+# This determines how many parallel tasks "make" is expecting, as it is
+# not exposed via an special variables, reserves them all, runs a subprocess
+# with PARALLELISM environment variable set, and releases the jobs back again.
+#
+# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
+from __future__ import print_function
+import os, sys, errno
+import subprocess
+
+# Extract and prepare jobserver file descriptors from envirnoment.
+claim = 0
+jobs = b""
+try:
+ # Fetch the make environment options.
+ flags = os.environ['MAKEFLAGS']
+
+ # Look for "--jobserver=R,W"
+ # Note that GNU Make has used --jobserver-fds and --jobserver-auth
+ # so this handles all of them.
+ opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
+
+ # Parse out R,W file descriptor numbers and set them nonblocking.
+ fds = opts[0].split("=", 1)[1]
+ reader, writer = [int(x) for x in fds.split(",", 1)]
+ # Open a private copy of reader to avoid setting nonblocking
+ # on an unexpecting process with the same reader fd.
+ reader = os.open("/proc/self/fd/%d" % (reader),
+ os.O_RDONLY | os.O_NONBLOCK)
+
+ # Read out as many jobserver slots as possible.
+ while True:
+ try:
+ slot = os.read(reader, 8)
+ jobs += slot
+ except (OSError, IOError) as e:
+ if e.errno == errno.EWOULDBLOCK:
+ # Stop at the end of the jobserver queue.
+ break
+ # If something went wrong, give back the jobs.
+ if len(jobs):
+ os.write(writer, jobs)
+ raise e
+ # Add a bump for our caller's reserveration, since we're just going
+ # to sit here blocked on our child.
+ claim = len(jobs) + 1
+except (KeyError, IndexError, ValueError, OSError, IOError) as e:
+ # Any missing environment strings or bad fds should result in just
+ # not being parallel.
+ pass
+
+# We can only claim parallelism if there was a jobserver (i.e. a top-level
+# "-jN" argument) and there were no other failures. Otherwise leave out the
+# environment variable and let the child figure out what is best.
+if claim > 0:
+ os.environ['PARALLELISM'] = '%d' % (claim)
+
+rc = subprocess.call(sys.argv[1:])
+
+# Return all the reserved slots.
+if len(jobs):
+ os.write(writer, jobs)
+
+sys.exit(rc)