diff options
author | Kris Kennaway <kris@FreeBSD.org> | 2008-07-19 17:59:41 +0000 |
---|---|---|
committer | Kris Kennaway <kris@FreeBSD.org> | 2008-07-19 17:59:41 +0000 |
commit | 54e565eedc284b13fe7c03fe05bd8a0ab3ff4fbf (patch) | |
tree | 2ed7567a9ca335eb974587ebb66efb39c72d218f /Mk/bsd.port.subdir.mk | |
parent | - Add support for display-name (diff) |
Major optimizations for 'make index' and other recursive traversal
targets.
* Use /rescue/sh for index builds instead of /bin/sh, when it exists.
The former is statically linked and faster to execute, which becomes
significant when executing it tens of thousands of times. This
trick can be used with other recursive targets by passing in
__MAKE_SHELL.
* Get rid of make variable assignments that use != command invocations
in the critical path, using several methods:
- rewriting logic to use shell or make builtins instead of external command executions
- macroizing commands and executing them in the targets where they
are needed instead of with every invocation of make
- precomputing the results of invariant commands in
bsd.port.subdir.mk and passing them in explicitly to child makes,
and using this to avoid recalculation in all the children. NB: the
commands are still run one per top-level subdirectory but this
does not currently seem to be a major issue. They could be moved
further up into the top-level Makefile at the cost of some
cleanliness.
- Committers are strongly discouraged from adding further "bare" !=
assignments to the ports tree, even in their own ports. One of
the above strategies should be used to avoid future bloat.
* Rewrite the core 'describe' target to work entirely within a single
shell process using only builtin commands. The old version is
retained as a backup for use on systems older than 603104, which
does not have the make :u modifier. This cuts down the number of
processes executed during the course of a 'make index' by an order
of magnitude, and we are essentially now amortized to the minimum of
a single make + sh instance per port, plus whatever commands the
port makefile itself executes (which are usually unnecessary and
bogus).
* Less validation of the WWW: target is performed; this can become
policed at a port level by portlint. Specifically we look at the
second word of the first line beginning with "WWW:" in pkg-descr,
and append "http://" to it unless it already begins with "http://",
"https://" or "ftp://". Thanks to dougb for the idea of how to
extract WWW: using shell builtins.
* Use the "true" shell builtin instead of echo > /dev/null for a
measurable decrease in CPU use.
* Add a note about dubious escaping strategy in bsd.port.subdir.mk
* Minor change in output of 'make describe': it no longer strips
trailing CR characters from pkg-descr files with MSDOS CR/LF
termination. Instead the makeindex perl script that post-processes
make describe into the INDEX is tweaked to strip on input.
The bottom line is that on my test hardware INDEX builds are now
faster by more than a factor of 2 and with a reduction in system time
by a factor of 4-8 depending on configuration.
Notes
Notes:
svn path=/head/; revision=217132
Diffstat (limited to 'Mk/bsd.port.subdir.mk')
-rw-r--r-- | Mk/bsd.port.subdir.mk | 82 |
1 files changed, 74 insertions, 8 deletions
diff --git a/Mk/bsd.port.subdir.mk b/Mk/bsd.port.subdir.mk index 08c1ab55af3d..5a7de96161ab 100644 --- a/Mk/bsd.port.subdir.mk +++ b/Mk/bsd.port.subdir.mk @@ -63,13 +63,15 @@ DESCR?= ${PKGDIR}/pkg-descr STRIP?= -s .endif +# These are variables that are invariant for the lifetime of a recursive port traversal +# (index build, etc), so it is more efficient to precompute them here and pass them in +# to child makes explicitly, instead of recomputing them tens of thousands of times. + .if !defined(NOPRECIOUSMAKEVARS) .if !defined(ARCH) ARCH!= ${UNAME} -p .endif -.if !defined(OSREL) -OSREL!= ${UNAME} -r | ${SED} -e 's/[-(].*//' -.endif + .if !defined(OSVERSION) .if exists(/usr/include/sys/param.h) OSVERSION!= ${AWK} '/^\#define[[:blank:]]__FreeBSD_version/ {print $$3}' < /usr/include/sys/param.h @@ -79,23 +81,76 @@ OSVERSION!= ${AWK} '/^\#define[[:blank:]]__FreeBSD_version/ {print $$3}' < /usr/ OSVERSION!= ${SYSCTL} -n kern.osreldate .endif .endif + +.if !defined(_OSRELEASE) +_OSRELEASE!= uname -r +.endif +.if !defined(OSREL) +OSREL= ${_OSRELEASE:C/[-(].*//} .endif -INDEXDIR?= ${PORTSDIR} -INDEXFILE?= INDEX-${OSVERSION:C/([0-9]).*/\1/} +.if !defined(OPSYS) +OPSYS!= ${UNAME} -s +.endif + +.if ${ARCH} == "amd64" || ${ARCH} =="ia64" +.if !defined(HAVE_COMPAT_IA32_KERN) +HAVE_COMPAT_IA32_KERN!= if ${SYSCTL} -n compat.ia32.maxvmem >/dev/null 2>&1; then echo YES; fi +.endif +.endif +.if !defined(CONFIGURE_MAX_CMD_LEN) +CONFIGURE_MAX_CMD_LEN!= ${SYSCTL} -n kern.argmax +.endif + +.if !defined(PYTHON_DEFAULT_VERSION) +PYTHON_DEFAULT_VERSION!= make -V PYTHON_DEFAULT_VERSION USE_PYTHON=1 -f ${PORTSDIR}/Mk/bsd.port.mk +.endif + +.if !defined(PYTHON_DEFAULT_PORTVERSION) +# We are caching the PYTHON_PORTVERSION of the default python version so we can reuse it in the +# common case. +PYTHON_DEFAULT_PORTVERSION!= make -V PYTHON_PORTVERSION USE_PYTHON=1 -f ${PORTSDIR}/Mk/bsd.port.mk +.endif + +.if !defined(PYTHONBASE) +PYTHONBASE!= make -V PYTHONBASE USE_PYTHON=1 -f ${PORTSDIR}/Mk/bsd.port.mk +.endif + +.if !defined(_JAVA_VERSION_LIST_REGEXP) +_JAVA_VERSION_LIST_REGEXP!= make -V _JAVA_VERSION_LIST_REGEXP USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk +.endif + +.if !defined(_JAVA_VENDOR_LIST_REGEXP) +_JAVA_VENDOR_LIST_REGEXP!= make -V _JAVA_VENDOR_LIST_REGEXP USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk +.endif + +.if !defined(_JAVA_OS_LIST_REGEXP) +_JAVA_OS_LIST_REGEXP!= make -V _JAVA_OS_LIST_REGEXP USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk +.endif + +.if !defined(_JAVA_PORTS_INSTALLED) +_JAVA_PORTS_INSTALLED!= make -V _JAVA_PORTS_INSTALLED USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk +.endif + +.if !defined(UID) UID!= ${ID} -u +.endif + .if exists(${LOCALBASE}/sbin/pkg_info) PKG_INFO?= ${LOCALBASE}/sbin/pkg_info .else PKG_INFO?= /usr/sbin/pkg_info .endif +.if !defined(PKGINSTALLVER) PKGINSTALLVER!= ${PKG_INFO} -P 2>/dev/null | ${SED} -e 's/.*: //' +.endif -.if !defined(OPSYS) -OPSYS!= ${UNAME} -s .endif +INDEXDIR?= ${PORTSDIR} +INDEXFILE?= INDEX-${OSVERSION:C/([0-9]).*/\1/} + # local customization of the ports tree .if exists(${.CURDIR}/Makefile.local) .include "${.CURDIR}/Makefile.local" @@ -307,6 +362,8 @@ README.html: > $@ @${RM} -f $@.tmp $@.tmp2 $@.tmp3 $@.tmp4 +# Pass in the cached invariant variables to child makes. +# XXX Why are we trying to escape these characters using regexps and not using ':Q'? .if !defined(NOPRECIOUSMAKEVARS) .MAKEFLAGS: \ ARCH="${ARCH:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \ @@ -314,7 +371,16 @@ README.html: OSREL="${OSREL:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \ OSVERSION="${OSVERSION:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \ UID="${UID:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \ - PKGINSTALLVER="${PKGINSTALLVER:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" + PKGINSTALLVER="${PKGINSTALLVER:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \ + HAVE_COMPAT_IA32_KERN="${HAVE_COMPAT_IA32_KERN}" \ + CONFIGURE_MAX_CMD_LEN="${CONFIGURE_MAX_CMD_LEN}" \ + PYTHON_DEFAULT_VERSION="${PYTHON_DEFAULT_VERSION}" \ + PYTHON_DEFAULT_PORTVERSION="${PYTHON_DEFAULT_PORTVERSION}" \ + PYTHONBASE="${PYTHONBASE}" \ + _JAVA_VERSION_LIST_REGEXP="${_JAVA_VERSION_LIST_REGEXP}" \ + _JAVA_VENDOR_LIST_REGEXP="${_JAVA_VENDOR_LIST_REGEXP}" \ + _JAVA_OS_LIST_REGEXP="${_JAVA_OS_LIST_REGEXP}" \ + _JAVA_PORTS_INSTALLED="${_JAVA_PORTS_INSTALLED}" .endif PORTSEARCH_DISPLAY_FIELDS?=name,path,info,maint,index,bdeps,rdeps,www |