Add chkdepschain.py - a tool to address one of the most annoying when it comes

down to user support flaws in the FreeBSD ports system.  The flaw in question
is related to the fact that dependencies are often "chained", which allows to
simplify maintenance of ports with large number of implied dependencies (a la
Evolution, Nautilus, you-name-it). Dependency chaining it's not a problem by
itself, but the fact that when building or installing a port the system doesn't
check chain integrity - it's only checks that dependencies explicitly
specified in port's Makefile are satisfied, which opens wide window for
various hard-trackable problems when one or more links in the middle of the
chain missed.

The idea behind the tool is quite simple - it should be executed right after
main dependency checking procedure, two times for each build - check build-time
chain before building the port (pre-pre-extract) and check run-time chain
before installing the port (pre-pre-install).  When executed, the tool checks
integrity of the specified chain (build-time, run-time or both) and reports all
errors, both fatal (dependency isn't installed) and non-fatal (dependency is
installed, but different version).

I've wrote this tool mostly to simplify maintenance of the GNOME ports, but
it doesn't contain anything GNOME-specific, so that it could be used in the
other parts of tree as well.

As an example I've added GNOME_VALIDATE_DEPS_CHAIN knob into bsd.gnome.mk (off
by default), which enables automatic chain validation for all ports that
USE_GNOMELIBS. This is a bit hackish, because I've used pre-extract and
pre-install targets - what we probably need is a generic way to plug various
custom tasks specified in bsd.xxx.mk (where xxx is kde, gnome, python, etc.)
into various parts of the build process (something like {pre,post}-pre-foo,
{pre,post}-post-foo springs into my mind).

The code is quite raw, so that I would appreciate any bug reports, patches,
suggestions, constructive critiquie and so on.

2 files changed, 315 insertions, 0 deletions

diff --git a/Mk/bsd.gnome.mk b/Mk/bsd.gnome.mk
index bdd4762714fd..336ebc24ac79 100644
--- a/Mk/bsd.gnome.mk
+++ b/Mk/bsd.gnome.mk
@@ -308,5 +308,26 @@ LIB_DEPENDS+=	panel_applet.5:${PORTSDIR}/x11/gnomecore
 PLIST_SUB+=		GNOME:="@comment " NOGNOME:="" DATADIR="share"
 .endif
 
+# Start of GNOME_VALIDATE_DEPS_CHAIN part.
+.if defined(USE_GNOMELIBS) && defined(GNOME_VALIDATE_DEPS_CHAIN)
+BUILD_DEPENDS+=	python:${PORTSDIR}/lang/python
+CHKDPCHN_CMD?=	${PORTSDIR}/Tools/scripts/chkdepschain.py
+CHKDPCHN_CACHE=	.chkdpchn.cache.${PKGNAME}
+
+.if !target(pre-extract)
+pre-extract::
+	@${ECHO_MSG} "===>  Validating build-time dependency chain for ${PKGNAME}"
+	@${MKDIR} ${WRKDIR}
+	@${CHKDPCHN_CMD} -b -s ${WRKDIR}/${CHKDPCHN_CACHE}
+.endif
+
+.if !target(pre-install)
+pre-install::
+	@${ECHO_MSG} "===>  Validating run-time dependency chain for ${PKGNAME}"
+	@${CHKDPCHN_CMD} -r -L ${WRKDIR}/${CHKDPCHN_CACHE}
+.endif
+.endif
+# End of GNOME_VALIDATE_DEPS_CHAIN part.
+
 .endif
 # End of use part.
diff --git a/Tools/scripts/chkdepschain.py b/Tools/scripts/chkdepschain.py
new file mode 100755
index 000000000000..446031196538
--- /dev/null
+++ b/Tools/scripts/chkdepschain.py
@@ -0,0 +1,294 @@
+#!/usr/local/bin/python
+
+import os, os.path, popen2, types, sys, getopt, pickle
+
+# Global constants and semi-constants
+PKG_DBDIR = '/var/db/pkg'
+PORTSDIR = '/usr/ports'
+ROOT_PORTMK = '/usr/share/mk/bsd.port.mk'
+PLIST_FILE = '+CONTENTS'
+ORIGIN_PREF = '@comment ORIGIN:'
+MAKEFILE = 'Makefile'
+MAKE = 'make'
+
+# Global variables
+#
+# PortInfo cache
+picache = {}
+
+# Useful aliases
+op_isdir = os.path.isdir
+op_join = os.path.join
+op_split = os.path.split
+op_abspath = os.path.abspath
+
+
+#
+# Query origin of specified installed package.
+#
+def getorigin(pkg):
+    plist = op_join(PKG_DBDIR, pkg, PLIST_FILE)
+    for line in open(plist).xreadlines():
+	if line.startswith(ORIGIN_PREF):
+	    origin = line[len(ORIGIN_PREF):].strip()
+	    break
+    else:
+	raise RuntimeError('%s: no origin recorded' % plist)
+
+    return origin
+
+
+#
+# Execute external command and return content of its stdout.
+#
+def getcmdout(cmdline, filterempty = 1):
+    pipe = popen2.Popen3(cmdline, 1)
+    results = pipe.fromchild.readlines()
+    for stream in (pipe.fromchild, pipe.tochild, pipe.childerr):
+        stream.close()
+
+    if pipe.wait() != 0:
+        if type(cmdline) is types.StringType:
+            cmdline = (cmdline)
+        raise IOError('%s: external command returned non-zero error code' % \
+          cmdline[0])
+
+    if filterempty != 0:
+	results = filter(lambda line: len(line.strip()) > 0, results)
+
+    return results
+
+
+#
+# For a specified path (either dir or makefile) query requested make(1)
+# variables and return them as a tuple in exactly the same order as they
+# were specified in function call, i.e. querymakevars('foo', 'A', 'B') will
+# return a tuple with a first element being the value of A variable, and
+# the second one - the value of B.
+#
+def querymakevars(path, *vars):
+    if op_isdir(path):
+	path = op_join(path, MAKEFILE)
+    dirname, makefile = op_split(path)
+    cmdline = [MAKE, '-f', makefile]
+    savedir = os.getcwd()
+    os.chdir(dirname)
+    try:
+        for var in vars:
+	    cmdline.extend(('-V', var))
+
+        results = map(lambda line: line.strip(), getcmdout(cmdline, 0))
+    finally:
+        os.chdir(savedir)
+
+    return tuple(results)
+
+
+def parsedeps(depstr):
+    return tuple(map(lambda dep: dep.split(':'), depstr.split()))
+
+
+#
+# For a specified port return either a new instance of the PortInfo class,
+# or existing instance from the cache.
+#
+def getpi(path):
+    path = op_abspath(path)
+    if not picache.has_key(path):
+        picache[path] = PortInfo(path)
+    return picache[path]
+
+
+#
+# Format text string according to requested constrains. Useful when you have
+# to display multi-line, variable width message on terminal.
+#
+def formatmsg(msg, wrapat = 78, seclindent = 0):
+    words = msg.split()
+    result = ''
+    isfirstline = 1
+    position = 0
+    for word in words:
+        if position + 1 + len(word) > wrapat:
+            result += '\n' + ' ' * seclindent + word
+            position = seclindent + len(word)
+            isfirstline = 0
+        else:
+            if position != 0:
+                result += ' '
+                position += 1
+            result += word
+            position += len(word)
+
+    return result
+
+
+#
+# Class that contain main info about the port
+#
+class PortInfo:
+    PKGNAME = None
+    CATEGORIES = None
+    MAINTAINER = None
+    BUILD_DEPENDS = None
+    LIB_DEPENDS = None
+    RUN_DEPENDS = None
+    PKGORIGIN = None
+    # Cached values, to speed-up things
+    __deps = None
+    __bt_deps = None
+    __rt_deps = None
+
+    def __init__(self, path):
+        self.PKGNAME, self.CATEGORIES, self.MAINTAINER, self.BUILD_DEPENDS, \
+          self.LIB_DEPENDS, self.RUN_DEPENDS, self.PKGORIGIN = \
+          querymakevars(path, 'PKGNAME', 'CATEGORIES', 'MAINTAINER', \
+          'BUILD_DEPENDS', 'LIB_DEPENDS', 'RUN_DEPENDS', 'PKGORIGIN')
+
+    def __str__(self):
+        return 'PKGNAME:\t%s\nCATEGORIES:\t%s\nMAINTAINER:\t%s\n' \
+          'BUILD_DEPENDS:\t%s\nLIB_DEPENDS:\t%s\nRUN_DEPENDS:\t%s\n' \
+          'PKGORIGIN:\t%s' % (self.PKGNAME, self.CATEGORIES, self.MAINTAINER, \
+          self.BUILD_DEPENDS, self.LIB_DEPENDS, self.RUN_DEPENDS, \
+          self.PKGORIGIN)
+
+    def getdeps(self):
+        if self.__deps == None:
+            result = []
+            for depstr in self.BUILD_DEPENDS, self.LIB_DEPENDS, \
+              self.RUN_DEPENDS:
+                deps = tuple(map(lambda dep: dep[1], parsedeps(depstr)))
+                result.append(deps)
+            self.__deps = tuple(result)
+        return self.__deps
+
+    def get_bt_deps(self):
+        if self.__bt_deps == None:
+            topdeps = self.getdeps()
+            topdeps = list(topdeps[0] + topdeps[1])
+            for dep in topdeps[:]:
+                botdeps = filter(lambda dep: dep not in topdeps, \
+                  getpi(dep).get_rt_deps())
+                topdeps.extend(botdeps)
+            self.__bt_deps = tuple(topdeps)
+        return self.__bt_deps
+        
+    def get_rt_deps(self):
+        if self.__rt_deps == None:
+            topdeps = self.getdeps()
+            topdeps = list(topdeps[1] + topdeps[2])
+            for dep in topdeps[:]:
+                botdeps = filter(lambda dep: dep not in topdeps, \
+                  getpi(dep).get_rt_deps())
+                topdeps.extend(botdeps)
+            self.__rt_deps = tuple(topdeps)
+        return self.__rt_deps
+
+
+def write_msg(*message):
+    if type(message) == types.StringType:
+        message = message,
+    message = tuple(filter(lambda line: line != None, message))
+    sys.stderr.writelines(message)
+
+
+#
+# Print optional message and usage information and exit with specified exit
+# code.
+#
+def usage(code, msg = None):
+    myname = os.path.basename(sys.argv[0])
+    if msg != None:
+        msg = str(msg) + '\n'
+    write_msg(msg, "Usage: %s [-rb] [-l|L cachefile] [-s cachefile]\n" % \
+      myname)
+    sys.exit(code)
+
+
+def main():
+    global picache
+
+    # Parse command line arguments
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'rbl:L:s:')
+    except getopt.GetoptError, msg:
+        usage(2, msg)
+
+    if len(args) > 0 or len(opts) == 0 :
+        usage(2)
+
+    cachefile = None
+    chk_bt_deps = 0
+    chk_rt_deps = 0
+    for o, a in opts:
+        if o == '-b':
+            chk_bt_deps = 1
+        elif o == '-r':
+            chk_rt_deps = 1
+        elif o in ('-l', '-L'):
+            # Try to load saved PortInfo cache
+            try:
+                picache = pickle.load(open(a))
+            except:
+                picache = {}
+            try:
+                if o == '-L':
+                    os.unlink(a)
+            except:
+                pass
+        elif o == '-s':
+            cachefile = a
+
+    # Load origins of all installed packages
+    instpkgs = os.listdir(PKG_DBDIR)
+    instpkgs = filter(lambda pkg: op_isdir(op_join(PKG_DBDIR, pkg)), instpkgs)
+    origins = {}
+    for pkg in instpkgs:
+        origins[pkg] = getorigin(pkg)
+
+    # Resolve dependencies for the current port
+    info = getpi(os.getcwd())
+    deps = []
+    if chk_bt_deps != 0:
+        deps.extend(filter(lambda d: d not in deps, info.get_bt_deps()))
+    if chk_rt_deps != 0:
+        deps.extend(filter(lambda d: d not in deps, info.get_rt_deps()))
+
+    # Perform validation
+    nerrs = 0
+    nwarns = 0
+    for dep in deps:
+        pi = getpi(dep)
+        if pi.PKGORIGIN not in origins.values():
+            print formatmsg(seclindent = 7 * 0, msg = \
+              'Error: package %s (%s) belongs to dependency chain, but ' \
+              'isn\'t installed.' % (pi.PKGNAME, pi.PKGORIGIN))
+            nerrs += 1
+        elif pi.PKGNAME not in origins.keys():
+            for instpkg in origins.keys():
+                if origins[instpkg] == pi.PKGORIGIN:
+                    break
+            print formatmsg(seclindent = 9 * 0, msg = \
+              'Warning: package %s (%s) belongs to dependency chain, but ' \
+              'package %s is installed instead. Perhaps it\'s an older ' \
+              'version - update is highly recommended.' % (pi.PKGNAME, \
+              pi.PKGORIGIN, instpkg))
+            nwarns += 1
+
+    # Save PortInfo cache if requested
+    if cachefile != None:
+        try:
+            pickle.dump(picache, open(cachefile, 'w'))
+        except:
+            pass
+
+    return nerrs
+
+
+PORTSDIR, PKG_DBDIR = querymakevars(ROOT_PORTMK, 'PORTSDIR', 'PKG_DBDIR')
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        pass