#!/usr/bin/python
#
# adt-run is part of autopkgtest
# autopkgtest is a tool for testing Debian binary packages
#
# autopkgtest is Copyright (C) 2006-2007, 2013 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# See the file CREDITS for a full list of credits information (often
# installed as /usr/share/doc/autopkgtest/CREDITS).

import signal
import optparse
import tempfile
import sys
import subprocess
import traceback
import urllib
import string
import re
import os
import errno
import fnmatch
import shutil
import copy
import time
import atexit

try:
    from debian import deb822, debian_support
    deb822, debian_support  # pyflakes
except ImportError:
    from debian_bundle import deb822, debian_support

from optparse import OptionParser

signal.signal(signal.SIGINT, signal.SIG_DFL)  # undo stupid Python SIGINT thing

try:
    our_base = os.environ['AUTOPKGTEST_BASE'] + '/lib'
except KeyError:
    our_base = '/usr/share/autopkgtest/python'
sys.path.insert(1, our_base)

from Autopkgtest import shellquote_cmdl

#---------- global variables

tmp = None		# pathstring on host
testbed = None		# Testbed
opts = None             # optparse options
errorcode = 0		# exit status that we are going to use
timeouts = {'short': 100, 'copy': 300, 'install': 3000, 'test': 10000, 'build': 100000}
binaries = None		# Binaries (.debs we have registered)
build_essential = ['build-essential']
os.putenv('APT_LISTBUGS_FRONTEND', 'none')
          # do not consider using apt-listbugs
os.putenv('APT_LISTCHANGES_FRONTEND', 'none')
          # do not consider using apt-listchanges


#
# logging/error functions
#

summary_stream = None


def error(message):
    '''Write an error message to stderr'''

    print >> sys.stderr, message


def debug(message, minlevel=0):
    '''Write a debug message to stderr according to --quiet'''

    if opts.debuglevel < minlevel:
        return

    p = 'adt-run'
    if minlevel:
        p += str(minlevel)
    p += ': '
    log_msg(message, p)


def log_msg(message, prefix=''):
    '''Write a string to stderr according to --quiet'''

    if opts.quiet:
        return
    for line in message.rstrip('\n').splitlines():
        s = prefix + line
        if not opts.quiet:
            print >> sys.stderr, s


def log_file(path, minlevel=0):
    '''Write a file to stderr according to --quiet'''

    if opts.debuglevel < minlevel or opts.quiet:
        return
    rc = subprocess.call(['cat', path], stdout=sys.stderr)
    if rc:
        bomb('cat failed for %s, exit code %d' % (path, rc))


def debug_subprocess(what, argv, script=None):
    '''Log a subprocess call for debugging'''

    o = '$ ' + what + ':'
    if argv is not None:
        ol = []
        for x in argv:
            if x is script:
                x = '<SCRIPT>'
            ol.append(x.replace('\\', '\\\\').replace(' ', '\\ '))
        o += ' ' + ' '.join(ol)
    debug(o)
    if script is not None and opts.debuglevel >= 1:
        o = ''
        for l in script.rstrip('\n').split('\n'):
            o += '$     ' + l + '\n'
        debug(o, 1)


def script_out(argv, what=None, script=None, **kwargs):
    '''Call a script and get its return code, and optionally stdout.

    If what/script are given, log these for debugging.

    Return (exitcode, stdout). stdout will be a string if kwargs contains
    stdout=subprocess.PIPE, otherwise None.
    '''
    if what:
        debug_subprocess(what, argv, script=script)

    process = subprocess.Popen(argv, **kwargs)
    output = process.communicate()[0]
    return (process.returncode, output)


def psummary(m):
    if summary_stream is not None:
        summary_stream.write(m.encode('UTF-8'))
        summary_stream.write('\n')


def preport(m):
    sys.stdout.write(m.encode('UTF-8'))
    sys.stdout.write('\n')
    sys.stdout.flush()
    psummary(m)


def report(tname, result):
    preport('%-20s %s' % (tname, result.decode('UTF-8', 'replace')))

#---------- errors we define


class Quit:

    def __init__(self, ec, m):
        self.ec = ec
        self.m = m


def bomb(m):
    raise Quit(20, 'unexpected error: %s' % m)


def badpkg(m):
    preport('blame: ' + ' '.join(testbed.blamed))
    preport('badpkg: ' + m)
    raise Quit(12, 'erroneous package: %s' % m)


class Unsupported(Exception):

    def __init__(self, lno, m):
        if lno >= 0:
            self.m = '%s (control line %d)' % (m, lno)
        else:
            self.m = m

    def report(self, tname):
        global errorcode
        errorcode |= 2
        report(tname, 'SKIP %s' % self.m)

#---------- convenience function


def mkdir_okexist(pathname, mode=02755):
    try:
        os.mkdir(pathname, mode)
    except (IOError, OSError), oe:
        if oe.errno != errno.EEXIST:
            raise


def rmtree(what, pathname):
    debug('/ %s rmtree %s' % (what, pathname), 2)
    try:
        shutil.rmtree(pathname)
    except (IOError, OSError), oe:
        if oe.errno not in (errno.EEXIST, errno.ENOENT):
            raise


def flatten(l):
    return reduce((lambda a, b: a + b), l, [])

#---------- parsing and representation of the arguments


class Action:

    def __init__(self, kind, arg, arghandling, what):
        # extra attributes get added during processing
        self.kind = kind
        self.arg = arg
        self.ah = arghandling
        self.what = what
        self.missing_tests_control = False

    def __repr__(self):
        return '<Action %s %s %s>' % (self.kind, self.what, self.arg)


def parse_args():
    global opts, timeouts
    global n_non_actions  # argh, stupid python scoping rules
    usage = '%prog <options> --- <virt-server>...'
    parser = OptionParser(usage=usage)

    arghandling = {
        'dsc_tests': True,
        'dsc_filter': '*',
        'deb_forbuilds': 'auto',
        'deb_fortests': 'auto',
        'set_lang': 'C'
    }
    initial_arghandling = arghandling.copy()
    n_non_actions = 0

    #----------
    # actions (ie, test sets to run, sources to build, binaries to use):

    def cb_action(op, optstr, value, parser, kindpath, is_act):
        global n_non_actions
        parser.largs.append((value, kindpath))
        n_non_actions += not(is_act)

    def pa_action(longname, metavar, kindpath, help, is_act=True):
        parser.add_option('--' + longname, action='callback', callback=cb_action,
                          nargs=1, type='string', callback_args=(kindpath, is_act),
                          help=help)

    pa_action('built-tree', 'TREE', '@/', help='run tests from build tree TREE')
    pa_action('unbuilt-tree', 'TREE', '@//', help='run tests from build tree TREE')

    pa_action('source', 'DSC', '@.dsc',
              help='build DSC and use its tests and/or generated binary packages')

    pa_action('binary', 'DEB', '@.deb',
              help='use binary package DEB according to most recent --binaries-* settings')

    pa_action('apt-source', 'SRCPKG', '@.apt',
              help='download with apt-get source in testbed and use its tests')

    def cb_actnoarg(op, optstr, value, parser, largsentry):
        parser.largs.append(largsentry)

    parser.add_option('--instantiate', action='callback', callback=cb_actnoarg,
                      callback_args=((None, ('instantiate',)),),
                      help='instantiate testbed now (during testing phase)'
                      ' and install packages'
                      ' selected for automatic installation, even'
                      ' if this might apparently not be required otherwise')

    #----------
    # argument handling settings (what ways to use action
    #  arguments, and pathname processing):

    def cb_setah(option, opt_str, value, parser, toset, setval):
        if type(setval) == list:
            if not value in setval:
                parser.error('value for %s option (%s) is not '
                             'one of the permitted values (%s)' %
                             (value, opt_str, setval.join(' ')))
        elif setval is not None:
            value = setval
        for v in toset:
            arghandling[v] = value
        parser.largs.append(arghandling.copy())

    def pa_setah(short, long, affected, effect, metavar=None, **kwargs):
        type = metavar
        if type is not None:
            type = 'string'
        parser.add_option(long, short, action='callback', callback=cb_setah,
                          callback_args=(affected, effect), **kwargs)

    #---- source processing settings:

    pa_setah(None, '--sources-tests', ['dsc_tests'], True,
             help='run tests from builds of subsequent sources')
    pa_setah(None, '--sources-no-tests', ['dsc_tests'], False,
             help='do not run tests from builds of subsequent sources')

    pa_setah(None, '--built-binaries-filter', ['dsc_filter'], None,
             type='string', metavar='PATTERN-LIST',
             help='from subsequent sources, use binaries matching'
             ' PATTERN-LIST (comma-separated glob patterns)'
             ' according to most recent --binaries-* settings')
    pa_setah('-B', '--no-built-binaries', ['dsc_filter'], '_',
             help='from subsequent sources, do not use any binaries')
    #---- binary package processing settings:

    def pa_setahbins(long, toset, how):
        pa_setah(None, long, toset, ['ignore', 'auto', 'install'],
                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
                 help=how + ' ignore binaries, install them as needed'
                 ' for dependencies, or unconditionally install'
                 ' them, respectively')
    pa_setahbins('--binaries', ['deb_forbuilds', 'deb_fortests'], '')
    pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
    pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')

    #----------
    # general options:

    def cb_vserv(op, optstr, value, parser):
        parser.values.vserver = list(parser.rargs)
        del parser.rargs[:]

    parser.add_option('--leave-lang', dest='set_lang', action='store_false',
                      help="leave LANG on testbed set to testbed's default")
    parser.add_option('--set-lang', metavar='LANGVAL',
                      help='set LANG on testbed to LANGVAL', default='C')

    parser.add_option('-o', '--output-dir',
                      help='write test artifacts (stdout/err, log, debs, etc.) '
                      'to OUTPUT-DIR, emptying it beforehand')
    # backwards compatible alias
    parser.add_option('--tmp-dir', dest='output_dir',
                      help='alias for --output-dir for backwards compatibility')

    parser.add_option('-l', '--log-file', dest='logfile',
                      help='write the log LOGFILE, emptying it beforehand,'
                      ' instead of using OUTPUT-DIR/log')
    parser.add_option('--summary-file', dest='summary',
                      help='write a summary report to SUMMARY, emptying it beforehand')

    for k in timeouts.keys():
        parser.add_option('--timeout-' + k, type='int', dest='timeout_' + k,
                          metavar='T', help='set %s timeout to T' % k)
    parser.add_option('--timeout-factor', type='float', metavar='FACTOR',
                      default=1.0,
                      help='multiply all default timeouts by FACTOR')

    parser.add_option('-u', '--user', help='run tests as USER (needs root on testbed)')
    parser.add_option('--gain-root', type='string', dest='gainroot',
                      help='prefix debian/rules binary with GAINROOT')
    parser.add_option('-q', '--quiet', action='store_true', dest='quiet', default=False)
    parser.add_option('-d', '--debug', action='count', dest='debuglevel', default=0)
    parser.add_option('--gnupg-home', type='string', dest='gnupghome',
                      default='~/.autopkgtest/gpg',
                      help='use GNUPGHOME rather than ~/.autopkgtest (for'
                      ' signing private apt archive);'
                      ' "fresh" means generate new key each time.')
    parser.add_option('--setup-commands', metavar='COMMANDS_OR_PATH',
                      help='Run these commands after opening the testbed '
                      '(e. g. "apt-get update" or adding apt sources); can be '
                      'a string with the commands, or a file containing the '
                      'commands')

    #----------
    # actual meat:

    class SpecialOption(optparse.Option):
        pass
    vs_op = SpecialOption('', '--VSERVER-DUMMY')
    vs_op.action = 'callback'
    vs_op.type = None
    vs_op.default = None
    vs_op.nargs = 0
    vs_op.callback = cb_vserv
    vs_op.callback_args = ()
    vs_op.callback_kwargs = {}
    vs_op.help = 'introduces virtualisation server and args'
    vs_op._short_opts = []
    vs_op._long_opts = ['---']

    parser.add_option(vs_op)

    (opts, args) = parser.parse_args()
    if not hasattr(opts, 'vserver'):
        parser.error('you must specifiy --- <virt-server>...')
    if n_non_actions >= len(parser.largs):
        parser.error('nothing to do specified')

    for k in timeouts.keys():
        t = getattr(opts, 'timeout_' + k)
        if t is None:
            t = timeouts[k] * opts.timeout_factor
        timeouts[k] = int(t)

    # this timeout is for adt-virt-*, so pass it down via environment
    os.environ['ADT_VIRT_COPY_TIMEOUT'] = str(timeouts['copy'])

    arghandling = initial_arghandling
    opts.actions = []
    ix = 0
    for act in args:
        if type(act) == dict:
            arghandling = act
            continue
        elif type(act) == tuple:
            pass
        elif type(act) == str:
            act = (act, act)
        else:
            raise Exception("unknown action in list `%s' having"
                            " type `%s'" % (act, type(act)))
        (pathstr, kindpath) = act

        if type(kindpath) is tuple:
            kind = kindpath[0]
        elif kindpath.endswith('.deb'):
            kind = 'deb'
        elif kindpath.endswith('.dsc'):
            kind = 'dsc'
        elif kindpath.endswith('.apt'):
            kind = 'apt'
        elif kindpath.endswith('//'):
            kind = 'ubtree'
        elif kindpath.endswith('/'):
            kind = 'tree'
        elif re.match('[0-9a-z][0-9a-z.+-]+', kindpath):
            kind = 'apt'
        else:
            parser.error("do not know how to handle filename `%s';"
                         " specify --source, --binary, --build-tree, or --apt-source" %
                         kindpath)

        what = '%s%s' % (kind, ix)
        ix += 1

        # "no built binaries" for apt sources
        if kind == 'apt':
            arghandling['dsc_filter'] = '_'

        opts.actions.append(Action(kind, pathstr, arghandling, what))


def setup_trace():
    global tmp, summary_stream

    if opts.output_dir is not None:
        rmtree('tmp(specified)', opts.output_dir)
        mkdir_okexist(opts.output_dir, 0755)
        tmp = opts.output_dir
    else:
        assert(tmp is None)
        tmp = tempfile.mkdtemp()
        os.chmod(tmp, 0755)

    if opts.logfile is None and opts.output_dir is not None:
        opts.logfile = opts.output_dir + '/log'

    if opts.logfile is not None:
        # tee stdout/err into log file
        fifo_log = os.path.join(tmp, 'fifo_log')
        os.mkfifo(fifo_log)
        atexit.register(os.unlink, fifo_log)
        out_tee = subprocess.Popen(['tee', fifo_log],
                                   stdin=subprocess.PIPE)
        err_tee = subprocess.Popen(['tee', fifo_log, '-a', '/dev/stderr'],
                                   stdin=subprocess.PIPE,
                                   stdout=open('/dev/null', 'wb'))
        subprocess.Popen(['cat', fifo_log], stdout=open(opts.logfile, 'wb'))
        os.dup2(out_tee.stdin.fileno(), sys.stdout.fileno())
        os.dup2(err_tee.stdin.fileno(), sys.stderr.fileno())
        atexit.register(out_tee.wait)
        atexit.register(out_tee.terminate)
        atexit.register(err_tee.wait)
        atexit.register(err_tee.terminate)

    if opts.summary is not None:
        summary_stream = open(opts.summary, 'w', 0)

    debug('options: ' + str(opts) + '; timeouts: ' + str(timeouts), 1)


def finalise_options():
    global opts, testbed, build_essential

    if opts.user is None and 'root-on-testbed' in testbed.caps:
        su = 'suggested-normal-user='
        ul = [
            e[len(su):]
            for e in testbed.caps
            if e.startswith(su)
        ]
        if ul:
            opts.user = ul[0]
        else:
            opts.user = ''

    if opts.user:
        if 'root-on-testbed' not in testbed.caps:
            error('warning: virtualisation system does not offer root on '
                  'testbed but --user option specified: failure likely')
        opts.user_wrap = lambda x: "su -s /bin/sh %s -c '%s'" % (opts.user, x)
    else:
        opts.user_wrap = lambda x: x

    if opts.gainroot is None:
        opts.gainroot = ''
        if (opts.user or
                'root-on-testbed' not in testbed.caps):
            opts.gainroot = 'fakeroot'
            build_essential += ['fakeroot']

    if opts.gnupghome.startswith('~/'):
        opts.gnupghome = os.path.expanduser(opts.gnupghome)
    elif opts.gnupghome == 'fresh':
        opts.gnupghome = None

    # if we have --setup-commands and it points to a file, read its contents
    if opts.setup_commands and os.path.exists(opts.setup_commands):
        with open(opts.setup_commands) as f:
            opts.setup_commands = f.read()

#---------- testbed management - the Testbed class


class Testbed:

    def __init__(self):
        self.sp = None
        self.lastsend = None
        self.scratch = None
        self.modified = False
        self.blamed = []
        self._debug('init', 1)
        self._need_reset_apt = False
        self.stop_sent = False
        self.ec_auxverbscript = None

    def _debug(self, m, minlevel=0):
        debug('** ' + m, minlevel)

    def start(self):
        self._debug('start', 1)
        debug_subprocess('vserver', opts.vserver)
        self.sp = subprocess.Popen(opts.vserver,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE)
        self.expect('ok')
        self.caps = self.commandr('capabilities')

    def stop(self):
        self._debug('stop', 1)
        if self.stop_sent:
            # avoid endless loop
            return
        self.stop_sent = True

        self.close()
        if self.sp is None:
            return
        ec = self.sp.returncode
        if ec is None:
            self.sp.stdout.close()
            self.send('quit')
            self.sp.stdin.close()
            ec = self.sp.wait()
        if ec:
            self.bomb('testbed gave exit status %d after quit' % ec)
        self.sp = None

    def open(self):
        self._debug('open, scratch=%s' % self.scratch, 1)
        if self.scratch is not None:
            return
        pl = self.commandr('open')
        self._opened(pl)

    def _opened(self, pl):
        self.scratch = pl[0]
        self.deps_processed = []
        self._auxverbscript_make()
        self.run_setup_commands()

    def _auxverbscript_make(self):
        pec = self.commandr('print-auxverb-command')
        if len(pec) < 1:
            self.bomb('too few results from print-execute-command')
        cmdl = map(urllib.unquote, pec[0].split(','))

        self._debug('cmdl = %s' % str(cmdl), 1)

        self.ec_auxverbscript = os.path.join(tmp, 'satdep-auxverb')
        with open(self.ec_auxverbscript, 'w') as f:
            f.write('''#!/bin/sh
set -e
if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
    # This is a pretty nasty hack.  Hopefully it can go away
    #  eventually.  See #635763.
    set -- dpkg --print-architecture
fi
export DEBIAN_FRONTEND=noninteractive
exec ''' + shellquote_cmdl(cmdl) + ' "$@"\n')
        os.chmod(self.ec_auxverbscript, 0755)

    def mungeing_apt(self):
        if not 'revert' in self.caps:
            self._need_reset_apt = True

    def reset_apt(self):
        if not self._need_reset_apt:
            return
        what = 'aptget-update-reset'
        cmdl = ['sh', '-c', 'apt-get -qy update 2>&1']
        rc = self.execute(what, cmdl, kind='install')
        if rc:
            error('\nwarning: failed to restore testbed apt cache, '
                  'exit code %d' % rc)
        what = 'aptconf-reset'
        cmdl = ['rm', '-f', '/etc/apt/apt.conf.d/90autopkgtest',
                '/etc/apt/sources.list.d/autopkgtest.list',
                '/etc/apt/preferences.d/90autopkgtest']
        rc = self.execute(what, cmdl, kind='install')
        if rc:
            error('\nwarning: failed to reset changes '
                  'made to testbed apt configuration, exit code %d' % rc)
        self._need_reset_apt = False

    def close(self):
        self._debug('close, scratch=%s' % self.scratch, 1)
        if self.ec_auxverbscript:
            os.unlink(self.ec_auxverbscript)
            self.ec_auxverbscript = None
        if self.scratch is None:
            return
        self.scratch = None
        if self.sp is None:
            return
        self.command('close')

    def run_setup_commands(self):
        if not opts.setup_commands:
            return

        debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ test bed setup')
        rc = testbed.execute('setup-commands',
                             ['sh', '-ec', opts.setup_commands],
                             script=opts.setup_commands,
                             so='/dev/stdout')
        if rc:
            bomb('testbed setup commands failed with status %i' % rc)

    def prepare1(self, deps_new):
        self._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
                    (self.modified, self.deps_processed, deps_new), 1)
        if 'revert' in self.caps and (
                self.modified or [d for d in self.deps_processed if d not in deps_new]):
            self._debug('reset **', 1)
            pl = self.commandr('revert')
            self._opened(pl)
        self.modified = False

    def prepare2(self, deps_new):
        self._debug('prepare2, deps_new=%s' % deps_new, 1)
        binaries.publish()
        self._install_deps(deps_new)

    def prepare(self, deps_new):
        self.prepare1(deps_new)
        self.prepare2(deps_new)

    def _install_deps(self, deps_new):
        self._debug(' installing dependencies ' + str(deps_new), 1)
        self.deps_processed = deps_new
        if not deps_new:
            return
        self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')

    def needs_reset(self):
        # show what caused a reset
        (fname, lineno, function, code) = traceback.extract_stack(limit=2)[0]
        self._debug('needs_reset, previously=%s, requested by %s() line %i' %
                    (self.modified, function, lineno), 1)
        self.modified = True

    def blame(self, m):
        self._debug('blame += %s' % m, 1)
        self.blamed.append(m)

    def bomb(self, m):
        self._debug('bomb %s' % m, 1)
        self.reset_apt()
        self.stop()
        raise Quit(16, 'testbed failed: %s' % m)

    def send(self, string):
        self.sp.stdin
        try:
            debug('>> ' + string, 2)
            print >>self.sp.stdin, string
            self.sp.stdin.flush()
            self.lastsend = string
        except:
            (type, value, dummy) = sys.exc_info()
            self.bomb('cannot send to testbed: %s' % traceback.
                      format_exception_only(type, value))

    def expect(self, keyword, nresults=None):
        l = self.sp.stdout.readline()
        if not l:
            self.bomb('unexpected eof from the testbed')
        if not l.endswith('\n'):
            self.bomb('unterminated line from the testbed')
        l = l.rstrip('\n')
        debug('<< ' + l, 2)
        ll = l.split()
        if not ll:
            self.bomb('unexpected whitespace-only line from the testbed')
        if ll[0] != keyword:
            if self.lastsend is None:
                self.bomb("got banner `%s', expected `%s...'" %
                          (l, keyword))
            else:
                self.bomb("sent `%s', got `%s', expected `%s...'" %
                          (self.lastsend, l, keyword))
        ll = ll[1:]
        if nresults is not None and len(ll) != nresults:
            self.bomb("sent `%s', got `%s' (%d result parameters),"
                      " expected %d result parameters" %
                      (self.lastsend, l, len(ll), nresults))
        return ll

    def commandr(self, cmd, args=(), nresults=None, unquote=True):
        # pass args=[None,...] or =(None,...) to avoid more url quoting
        if type(cmd) is str:
            cmd = [cmd]
        if len(args) and args[0] is None:
            args = args[1:]
        else:
            args = map(urllib.quote, args)
        al = cmd + args
        self.send(string.join(al))
        ll = self.expect('ok', nresults)
        if unquote:
            ll = map(urllib.unquote, ll)
        return ll

    def command(self, cmd, args=()):
        self.commandr(cmd, args, 0)

    def commandr1(self, cmd, args=()):
        rl = self.commandr(cmd, args, 1)
        return rl[0]

    def execute(self, what, cmdl,
                si='/dev/null', so='/dev/null', se=None, cwd=None,
                script=False, xenv=[], kind='short'):
        # Options for script:
        #   False - do not call debug_subprocess, no synch. reporting required
        #   None or string - call debug_subprocess with that value,
        #			plumb stderr through synchronously if possible
        # Options for se:
        #   None - usual logging (output is of kind 2c)
        #   string - path on testbed (output is of kind 2d)

        timeout = timeouts[kind]

        if script is not False:
            debug_subprocess(what, cmdl, script=script)
        if cwd is None:
            cwd = self.scratch

        xdump = None
        if se is None:
            se_catch = TempTestbedPath(testbed, what + '-xerr')
            se_use = se_catch.tb
            if not opts.quiet:
                xdump = 'debug=2-2'
        else:
            se_catch = None
            se_use = se

        cmdl = [None,
                ','.join(map(urllib.quote, cmdl)),
                si, so, se_use, cwd]

        if timeout is not None and timeout > 0:
            cmdl.append('timeout=%d' % timeout)

        if xdump is not None and 'execute-debug' in self.caps:
            cmdl += [xdump]
        for e in xenv:
            cmdl.append('env=%s' % e)
        if kind == 'install':
            cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
        if opts.set_lang is not False:
            cmdl.append('env=LANG=%s' % opts.set_lang)

        rc = self.commandr1('execute', cmdl)
        try:
            rc = int(rc)
        except ValueError:
            bomb("execute for %s gave invalid response `%s'"
                 % (what, rc))

        if se_catch is not None:
            se_catch.copyup()
            log_file(se_catch.host)

        return rc

    def satisfy_dependencies_string(self, deps, what):
        # Must have called Binaries.configure_apt
        debug('dependencies: %s: satisfying %s' % (what, deps), 1)
        dsc = os.path.join(tmp, 'deps.dsc')
        with open(dsc, 'w') as f:
            f.write('Build-Depends: %s\n\n' % deps)
        # pbuilder-satisfydepends has a bug where it ignores the
        # Build-Depends if it's the last line in the dsc (#635696)
        self.satisfy_dependencies_dsc(dsc, what)
        os.unlink(dsc)

    def satisfy_dependencies_dsc(self, dsc, what):
        # Must have called Binaries.configure_apt
        cmdl = ['/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
                '--binary-all',  # --check-key
                '--internal-chrootexec', self.ec_auxverbscript,
                '-c', dsc
                ]
        # The --internal-chrootexec option is really handy but
        # perhaps we are not supposed to use it ?  See also #635697.
        debug('dependencies: %s: running %s' % (what, str(cmdl)))
        rc = subprocess.call(cmdl, stdout=None, stderr=None)
        if rc:
            badpkg('dependency install failed, exit code %d' % rc)


class TestbedPath:
    '''Represent a file/dir with a host and a testbed path'''

    def __init__(self, testbed, host, tb, is_dir=None):
        '''Create a TestbedPath object.

        The object itself is just a pair of file names, nothing more. They do
        not need to exist until you call copyup() or copydown() on them.

        testbed: the Testbed object which this refers to
        host: path of the file on the host
        tb: path of the file in testbed
        is_dir: whether path is a directory; None for "unspecified" if you only
                need copydown()
        '''
        self.testbed = testbed
        self.host = host
        self.tb = tb
        self.is_dir = is_dir

    def copydown(self):
        '''Copy file from the host to the testbed'''

        # create directory on testbed
        rc = testbed.execute('mkdir',
                             ['sh', '-ec', 'test -d "$1" || mkdir -p "$1"',
                              'x', os.path.dirname(self.tb)])
        if rc:
            bomb('failed to create directory %s' % os.path.dirname(self.tb))

        if os.path.isdir(self.host):
            # directories need explicit '/' appended for VirtSubproc
            testbed.command('copydown', (self.host + '/', self.tb + '/'))
        else:
            testbed.command('copydown', (self.host, self.tb))

    def copyup(self):
        '''Copy file from the testbed to the host'''

        mkdir_okexist(os.path.dirname(self.host))
        assert self.is_dir is not None
        if self.is_dir:
            testbed.command('copyup', (self.tb + '/', self.host + '/'))
        else:
            testbed.command('copyup', (self.tb, self.host))


class TempTestbedPath(TestbedPath):
    '''Represent a file in the hosts'/testbed's temporary directories'''

    def __init__(self, testbed, name, is_dir=False, autoclean=True):
        '''Create a temporary TestbedPath object.

        The object itself is just a pair of file names, nothing more. They do
        not need to exist until you call copyup() or copydown() on them.

        testbed: the Testbed object which this refers to
        name: name of the temporary file (without path); host and tb
              will then be derived from that
        is_dir: whether path is a directory; None for "unspecified" if you only
                need copydown()
        autoclean: If True (default), remove file when adt-run finishes. Should
                be set to False for files which you want to keep in the
                --output-dir which are useful for reporting results, like test
                stdout/err, log files, and binaries.
        '''
        TestbedPath.__init__(self, testbed, os.path.join(tmp, name),
                             os.path.join(testbed.scratch, name),
                             is_dir)
        self.autoclean = autoclean

    def __del__(self):
        if self.autoclean:
            if os.path.exists(self.host):
                os.unlink(self.host)

#---------- representation of test control files: Field*, Test, etc.


class FieldBase:

    def __init__(self, fname, stz, base, tnames, vl):
        assert(vl)
        self.stz = stz
        self.fname = fname
        self.base = base
        self.tnames = tnames
        self.vl = vl

    def words(self):
        def distribute(vle):
            (lno, v) = vle
            r = v.split()
            r = map((lambda w: (lno, w)), r)
            return r
        return flatten(map(distribute, self.vl))

    def atmostone(self):
        if len(self.vl) == 1:
            (self.lno, self.v) = self.vl[0]
        else:
            raise Unsupported(self.vl[1][0],
                              'only one %s field allowed' % self.fname)
        return self.v


class FieldIgnore(FieldBase):

    def parse(self):
        pass


class Restriction:

    def __init__(self, rname, base):
        pass


class Restriction_rw_build_tree(Restriction):
    pass


class Restriction_build_needed(Restriction):
    pass


class Restriction_allow_stderr(Restriction):
    pass


class Restriction_breaks_testbed(Restriction):

    def __init__(self, rname, base):
        if 'revert-full-system' not in testbed.caps:
            raise Unsupported(-1, 'Test breaks testbed but testbed does not '
                              'advertise revert-full-system')


class Restriction_needs_root(Restriction):

    def __init__(self, rname, base):
        if 'root-on-testbed' not in testbed.caps:
            raise Unsupported(-1,
                              'Test needs root on testbed which is not available')


class Field_Restrictions(FieldBase):

    def parse(self):
        for wle in self.words():
            (lno, rname) = wle
            nrname = rname.replace('-', '_')
            try:
                rclass = globals()['Restriction_' + nrname]
            except KeyError:
                raise Unsupported(lno,
                                  'unknown restriction %s' % rname)
            r = rclass(nrname, self.base)
            self.base['restriction_names'].append(rname)
            self.base['restrictions'].append(r)


class Field_Features(FieldIgnore):

    def parse(self):
        for wle in self.words():
            (lno, fname) = wle
            self.base['feature_names'].append(fname)
            nfname = fname.replace('-', '_')
            try:
                fclass = globals()['Feature_' + nfname]
            except KeyError:
                continue
            ft = fclass(nfname, self.base)
            self.base['features'].append(ft)


class Field_Tests(FieldIgnore):
    pass


class Field_Depends(FieldBase):

    def parse(self):
        debug('Field_Depends: %s %s %s %s' %
              (self.stz, self.base, self.tnames, self.vl), 2)
        dl = map(lambda x: x.strip(),
                 flatten(map(lambda (lno, v): v.split(','), self.vl)))
        # Remove empty dependencies
        dl = filter(None, dl)

        dep_re = re.compile(
            r'(?P<package>[a-z0-9+-.]+)\s*(\((?P<relation><<|<=|>=|=|>>)\s*(?P<version>[^\)]*)\))?(\s*\[[[a-z0-9+-. ]+\])?$')
        for di in dl:
            for d in di.split('|'):
                d = d.strip()
                if d == '@':
                    continue  # Expanded to binary packages
                if d == '@builddeps@':
                    continue  # Expanded to build dependencies
                m = dep_re.match(d)
                if not m:
                    badpkg("Test Depends field contains an invalid "
                           "dependency `%s'" % d)
                if m.group("version"):
                    try:
                        debian_support.NativeVersion(m.group('version'))
                    except ValueError:
                        badpkg("Test Depends field contains dependency"
                               " `%s' with an invalid version" % d)
                    except AttributeError:
                        # too old python-debian, skip the check
                        pass
        self.base['depends'] = dl


class Field_Tests_directory(FieldBase):

    def parse(self):
        td = self.atmostone()
        if td.startswith('/'):
            raise Unsupported(self.lno,
                              'Tests-Directory may not be absolute')
        self.base['testsdir'] = td


def run_tests(stanzas, tree):
    global errorcode, testbed
    if stanzas == ():
        report('*', 'SKIP no tests in this package')
        errorcode |= 8
    for stanza in stanzas:
        tests = stanza[' tests']
        if not tests:
            report('*', 'SKIP package has metadata but no tests')
            errorcode |= 8
        for t in tests:
            t.prepare(tree)
            t.run(tree)
            if 'breaks-testbed' in t.restriction_names:
                testbed.needs_reset()
        testbed.needs_reset()


class Test:

    def __init__(self, tname, base, act):
        if '/' in tname:
            raise Unsupported(base[' lno'],
                              'test name may not contain / character')
        for k in base:
            setattr(self, k, base[k])
        self.tname = tname
        self.act = act
        self.what = act.what + 't-' + tname
        if len(base['testsdir']):
            self.path = base['testsdir'] + '/' + tname
        else:
            self.path = tname
        debug('constructed; path=%s' % self.path, 1)
        debug(' .depends=%s' % self.depends, 1)

    def _debug(self, m):
        debug('& %s: %s' % (self.what, m))

    def report(self, m):
        report(self.what, m)

    def reportfail(self, m):
        global errorcode
        errorcode |= 4
        report(self.what, 'FAIL ' + m)

    def prepare(self, tree):
        self._debug('preparing')
        dn = []
        for d in self.depends:
            self._debug(' processing dependency ' + d)
            if d == '@':
                # expand to all binaries from that source
                for pkg in packages_from_source(self.act, tree):
                    dp = d.replace('@', pkg)
                    self._debug('  synthesised dependency ' + dp)
                    dn.append(dp)
            elif d == '@builddeps@':
                for dp in build_deps_from_source(self.act, tree):
                    self._debug('  synthesised dependency ' + dp)
                    dn.append(dp)
            else:
                self._debug('  literal dependency ' + d)
                dn.append(d)
        testbed.prepare(dn)

    def run(self, tree):
        self._debug('[----------------------------------------')

        # ensure our tests are in the testbed
        tree.copydown()
        if opts.user:
            rc = testbed.execute('testchown-' + self.what, ['chown', '-R', opts.user, '--', tree.tb])
            if rc:
                bomb('failed to chown %s' % tree.tb)

        # stdout/err files in testbed
        so = TempTestbedPath(testbed, self.what + '-stdout', autoclean=False)
        se = TempTestbedPath(testbed, self.what + '-stderr', autoclean=False)

        tb_test_path = os.path.join(tree.tb, self.path)
        xenv = []

        rc = testbed.execute('testchmod-' + self.what, ['chmod', '+x', '--', tb_test_path])
        if rc:
            bomb('failed to chmod +x %s' % tb_test_path)

        testtmp = '%s/%s-testtmp' % (testbed.scratch, self.what)

        script = 'buildtree="$1"; shift\n'
        script += 'rm -rf -- "$@"; mkdir -m 755 -- "$@"\n'

        if 'needs-root' not in self.restriction_names and opts.user is not None:
            if 'root-on-testbed' not in testbed.caps:
                bomb('cannot change to user %s without root-on-testbed' % opts.user)
            test_argv = ['su', '-s', '/bin/sh', opts.user, '-c', tb_test_path]
            if opts.user:
                script += 'chown %s "$@"\n' % opts.user
                if 'rw-build-tree' in self.restriction_names:
                    script += ('chown -R %s "$buildtree"\n'
                               % opts.user)
        else:
            test_argv = [tb_test_path]

        test_tmpdir = testtmp + '/tmpdir'
        xenv.append('TMPDIR=%s' % test_tmpdir)
        script += 'chmod 1777 "%s"\n' % test_tmpdir

        test_adttmp = testtmp + '/adttmp'
        xenv.append('ADTTMP=%s' % test_adttmp)

        test_artifacts = testtmp + '/artifacts'
        script += 'mkdir -m 1777 "%s"\n' % test_artifacts
        xenv.append('ADT_ARTIFACTS=%s' % test_artifacts)

        rc = testbed.execute('mktmpdir-' + self.what,
                             ['sh', '-ec', script, 'x', tree.tb,
                              testtmp, test_tmpdir, test_adttmp,
                              ])
        if rc:
            bomb("could not create test tmp dirs in `%s', exit code %d"
                 % (testtmp, rc))

        # With the NULL runner the test and the controller both run on the same
        # file system and can route stdout/stderr through a common FIFO to
        # watch them in realtime. FIXME: Make this work for schroot/lxc/etc. if
        # they use a bind-mounted /tmp.
        # Note that we can't use the coreutils tee with pipes, as it aborts on
        # the first read error.
        #show_realtime = os.path.isdir(testtmp) and os.access(testtmp, os.W_OK)
        show_realtime = opts.vserver[0].endswith('-null')
        if show_realtime:
            fifo_out = os.path.join(testtmp, 'test_stdout')
            fifo_err = os.path.join(testtmp, 'test_stderr')
            os.mkfifo(fifo_out)
            os.mkfifo(fifo_err)
            debug('teeing to stdout: %s, stderr: %s' % (fifo_out, fifo_err), 2)
            tee_out = os.fork()
            if tee_out == 0:
                fd_in = os.open(fifo_out, os.O_RDONLY)
                fd_out = os.open(so.tb, os.O_CREAT | os.O_WRONLY)

                while True:
                    block = os.read(fd_in, 1024)
                    if not block:
                        break
                    os.write(fd_out, block)
                    os.write(1, block)
                os._exit(0)

            tee_err = os.fork()
            if tee_err == 0:
                fd_in = os.open(fifo_err, os.O_RDONLY)
                fd_out = os.open(se.tb, os.O_CREAT | os.O_WRONLY)

                while True:
                    block = os.read(fd_in, 1024)
                    if not block:
                        break
                    os.write(fd_out, block)
                    os.write(2, block)
                os._exit(0)

            try:
                rc = testbed.execute('test-' + self.what, test_argv,
                                     so=fifo_out, se=fifo_err, cwd=tree.tb,
                                     xenv=xenv, kind='test')
                debug('testbed executing test finished with exit status %i' % rc, 1)
            finally:
                # give tee childs another second to mop up last output
                time.sleep(1)
                os.kill(tee_out, signal.SIGTERM)
                os.kill(tee_err, signal.SIGTERM)
                os.waitpid(tee_out, 0)
                os.waitpid(tee_err, 0)

            os.unlink(fifo_out)
            os.unlink(fifo_err)

        # if testtmp is not accessible, we cannot show realtime output and need
        # to show stdout afterwards
        else:
            rc = testbed.execute('test-' + self.what, test_argv,
                                 so=so.tb, se=se.tb, cwd=tree.tb,
                                 xenv=xenv, kind='test')
            debug('testbed executing test finished with exit status %i' % rc, 1)

        self._debug('----------------------------------------]')

        # copy stdout/err files to host
        so.copyup()
        se.copyup()
        se_size = os.path.getsize(se.host)

        # avoid mixing up stdout (from report) and stderr (from _debug) in output
        sys.stdout.flush()
        sys.stderr.flush()
        time.sleep(0.1)

        self._debug(' - - - - - - - - - - results - - - - - - - - - -')

        if rc != 0:
            self.reportfail('non-zero exit status %d' % rc)
        elif se_size != 0 and 'allow-stderr' not in self.restriction_names:
            with open(se.host) as f:
                stderr_top = f.readline().rstrip('\n \t\r')
            self.reportfail('status: %d, stderr: %s' % (rc, stderr_top))
        else:
            self.report('PASS')

        # avoid mixing up stdout (from report) and stderr (from _debug) in output
        sys.stdout.flush()
        sys.stderr.flush()
        time.sleep(0.1)

        if os.path.getsize(so.host) != 0 and not show_realtime:
            self._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
            log_file(so.host)

        if se_size != 0 and (not show_realtime or 'allow-stderr'
                             not in self.restriction_names):
            self._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
            log_file(se.host)

        # copy artifacts to host, if we have --output-dir
        if opts.output_dir:
            ap = TestbedPath(testbed,
                             os.path.join(opts.output_dir, 'artifacts'),
                             test_artifacts, is_dir=True)
            ap.copyup()
            # don't keep an empty artifacts dir around
            if not os.listdir(ap.host):
                os.rmdir(ap.host)


def read_stanzas(path):
    stanzas = []

    try:
        control = open(path, 'r')
    except (IOError, OSError), oe:
        if oe[0] != errno.ENOENT:
            raise
        return []

    lno = 0
    stz = {}  # stz[field_name][index] = (lno, value)
                # special field names:
                # stz[' lno'] = number
                # stz[' tests'] = list of Test objects
                # empty dictionary means we're between stanzas
    for paragraph in deb822.Deb822.iter_paragraphs(control):
        lno += 1
        stz = {' lno': lno, ' tests': []}
        for field, value in paragraph.iteritems():
            v = ''.join(value.split('\n')).replace('  ', ' ')
            field = string.capwords(field)
            stz[field] = [(lno, v)]
            lno += 1 + value.count('\n')  # Count multilines fields
        stanzas.append(stz.copy())

    return stanzas


def read_control(act, control_path):

    if act.missing_tests_control:
        return ()
    stanzas = read_stanzas(control_path)

    for stz in stanzas:
        try:
            try:
                tnames = stz['Tests']
            except KeyError:
                tnames = ['*']
                raise Unsupported(stz[' lno'],
                                  'no Tests field')
            tnames = map((lambda lt: lt[1]), tnames)
            tnames = string.join(tnames).split()
            base = {
                'restriction_names': [],
                'restrictions': [],
                'feature_names': [],
                'features': [],
                'testsdir': 'debian/tests',
                'depends': '@',
                ' lno': stz[' lno'],
            }
            for fname in stz.keys():
                if fname.startswith(' '):
                    continue
                vl = stz[fname]
                try:
                    fclass = globals()['Field_' +
                                       fname.replace('-', '_')]
                except KeyError:
                    raise Unsupported(vl[0][0],
                                      'unknown metadata field %s' % fname)
                f = fclass(stz, fname, base, tnames, vl)
                f.parse()
            for tname in tnames:
                try:
                    t = Test(tname, base, act)
                    stz[' tests'].append(t)
                except Unsupported, u:
                    u.report(tname)
                    continue
        except Unsupported, u:
            for tname in tnames:
                u.report(tname)
            continue

    return stanzas


def print_exception(ei, msgprefix=''):
    if msgprefix:
        error(msgprefix)
    (et, q, tb) = ei
    if et is Quit:
        error('adt-run: ' + q.m)
        psummary('quitting: ' + q.m)
        return q.ec
    else:
        error('adt-run: unexpected, exceptional, error:')
        psummary('quitting: unexpected error, consult transcript')
        traceback.print_exc(None, sys.stderr)
        return 20


def cleanup():
    try:
        if testbed is not None:
            testbed.reset_apt()
            testbed.stop()
        if opts.output_dir is None and tmp is not None:
            rmtree('tmp', tmp)
    except:
        print_exception(sys.exc_info(),
                        '\nadt-run: error cleaning up:\n')
        sys.exit(20)

#---------- registration, installation etc. of .deb's: Binaries


def determine_package(act):
    cmd = 'dpkg-deb --info --'.split(' ') + [act.arg, 'control']
    (rc, output) = script_out(cmd, stdout=subprocess.PIPE)
    if rc:
        badpkg('failed to parse binary package, code %d' % rc)
    pkg_re = re.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
    act.pkg = None
    for l in output.split('\n'):
        m = pkg_re.match(l)
        if not m:
            continue
        if act.pkg:
            badpkg('two Package: lines in control file')
        act.pkg = m.groups()[0]
    if not act.pkg:
        badpkg('no good Package: line in control file')


def packages_from_source(act, tree):
    (rc, output) = script_out(['dh_listpackages'], stdout=subprocess.PIPE, cwd=tree.host)
    if rc:
        badpkg('failed to parse packages built from source, code %d' % rc)

    # filter out empty lines
    packages = [p for p in output.split() if p]

    # filter out udebs
    for st in read_stanzas(os.path.join(tree.host, 'debian/control')):
        if 'Package' not in st:
                    # source stanza
            continue
        if 'Xc-package-type' in st:
            try:
                packages.remove(st['Package'][0][1])
            except ValueError:
                pass

    return packages


def build_deps_from_source(act, tree):
    deps = []
    for st in read_stanzas(os.path.join(tree.host, 'debian/control')):
        if 'Build-depends' in st:
            for d in st['Build-depends'][-1][1].split(','):
                dp = d.strip()
                if dp:
                    deps.append(dp)
        if 'Build-depends-indep' in st:
            for d in st['Build-depends-indep'][-1][1].split(','):
                dp = d.strip()
                if dp:
                    deps.append(dp)
    return deps


class Binaries:

    def __init__(self, tb):
        self.dir = TempTestbedPath(testbed, 'binaries', is_dir=True, autoclean=False)
        os.mkdir(self.dir.host)
        ok = False

        if opts.gnupghome is None:
            opts.gnupghome = tmp + '/gnupg'

        self._debug('initialising')
        try:
            for x in ['pubring', 'secring']:
                os.stat(opts.gnupghome + '/' + x + '.gpg')
            ok = True
        except (IOError, OSError), oe:
            if oe.errno != errno.ENOENT:
                raise

        if ok:
            self._debug('no key generation needed')
        else:
            self.genkey()

        self.apt_get_cmd = 'apt-get -q -o Debug::pkgProblemResolver=true ' \
            '-o APT::Get::force-yes=true -o APT::Get::Assume-Yes=true'

    def __del__(self):
        # clean up an empty binaries output dir
        try:
            os.rmdir(self.dir.host)
        except OSError:
            pass

    def _debug(self, s):
        debug('* ' + s, 1)

    def genkey(self):
        self._debug('preparing for key generation')

        mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
        mkdir_okexist(opts.gnupghome, 0700)

        script = '''
  exec >&2
  cd "$1"
  cat <<"END" >key-gen-params
Key-Type: DSA
Key-Length: 1024
Key-Usage: sign
Name-Real: autopkgtest per-run key
Name-Comment: do not trust this key
Name-Email: autopkgtest@example.com
END
  set -x
  gpg --homedir="$1" --batch --gen-key key-gen-params
'''
        cmdl = ['sh', '-ec', script, 'x', opts.gnupghome]
        rc = script_out(cmdl, what='genkey', script=script)[0]
        if rc:
            bomb('key generation failed, code %d' % rc)

    def _configure_apt(self, tb):
        prefs = TestbedPath(testbed, os.path.join(tmp, 'apt-prefs'),
                            '/etc/apt/preferences.d/90autopkgtest')
        with open(prefs.host, 'w') as f:
            f.write('''Package: *
Pin: origin ""
Pin-Priority: 1002
''')
        prefs.copydown()
        os.unlink(prefs.host)

    def reset(self):
        self._debug('reset')
        rmtree('binaries', self.dir.host)
        os.mkdir(self.dir.host)
        self.install = []
        self.blamed = []
        self.registered = set()

    def register(self, act, pkg, path, forwhat, blamed):
        self._debug('register what=%s deb_%s=%s pkg=%s path=%s' %
                    (act.what, forwhat, act.ah['deb_' + forwhat], pkg, path))

        if act.ah['deb_' + forwhat] == 'ignore':
            return

        self.blamed += testbed.blamed

        dest = os.path.join(self.dir.host, pkg + '.deb')

        try:
            os.remove(dest)
        except (IOError, OSError), oe:
            if oe.errno != errno.ENOENT:
                raise oe

        try:
            os.link(path, dest)
        except (IOError, OSError), oe:
            if oe.errno != errno.EXDEV:
                raise oe
            shutil.copy(path, dest)
        # clean up locally built debs (what=ubtreeN) to keep a clean
        # --output-dir, but don't clean up --binary arguments
        if opts.output_dir and path.startswith(opts.output_dir):
            atexit.register(lambda f: os.path.exists(f) and os.unlink(f), path)

        if act.ah['deb_' + forwhat] == 'install':
            self.install.append(pkg)

        self.registered.add(pkg)

    def publish(self):
        self._debug('publish')
        if not self.registered:
            self._debug('no registered binaries, not publishing anything')
            return

        self._configure_apt(testbed)

        script = '''
  exec >&2
  cd "$1"
  apt-ftparchive packages . >Packages
  gzip <Packages >Packages.gz
  apt-ftparchive release . >Release
  rm -f Release.gpg
  gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
  gpg --homedir="$2" --batch --export >archive-key.pgp
'''
        cmdl = ['sh', '-ec', script, 'x', self.dir.host, opts.gnupghome]
        rc = script_out(cmdl, what='ftparchive', script=script)[0]
        if rc:
            bomb('apt-ftparchive or signature failed, code %d' % rc)

        # copy binaries directory to testbed
        self.dir.copydown()

        aptkey_out = TempTestbedPath(testbed, 'apt-key.out')
        script = '''
  exec 3>&1 >&2
  apt-key add archive-key.pgp
  echo "deb file://''' + self.dir.tb + ''' /" >/etc/apt/sources.list.d/autopkgtest.list
  if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
    echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
  fi
''' + self.apt_get_cmd + ''' update >&2
  cat /var/lib/dpkg/status >&3
'''
        testbed.mungeing_apt()
        rc = testbed.execute('apt-key', ['sh', '-ec', script],
                             so=aptkey_out.tb, cwd=self.dir.tb,
                             script=script, kind='install')
        if rc:
            bomb('apt setup failed with exit code %d' % rc)

        testbed.blamed += self.blamed

        aptkey_out.copyup()

        self._debug('publish reinstall checking...')
        pkgs_reinstall = set()
        pkg = None
        for l in open(aptkey_out.host):
            if l.startswith('Package: '):
                pkg = l[9:].rstrip()
            elif l.startswith('Status: install '):
                if pkg in self.registered:
                    pkgs_reinstall.add(pkg)
                    self._debug(' publish reinstall needs ' + pkg)

        if pkgs_reinstall:
            for pkg in pkgs_reinstall:
                testbed.blame(pkg)
            what = 'apt-get-reinstall'
            cmdl = (self.apt_get_cmd + ' --reinstall install ' +
                    ' '.join([pkg for pkg in pkgs_reinstall]) + ' >&2')
            cmdl = ['sh', '-c', cmdl]
            rc = testbed.execute(what, cmdl, script=None, kind='install')
            if rc:
                badpkg('installation of basic binaries failed,'
                       ' exit code %d' % rc)

        self._debug('publish install...')
        for pkg in self.install:
            what = 'apt-get-install-%s' % pkg
            testbed.blame(pkg)
            cmdl = self.apt_get_cmd + ' install ' + pkg + ' >&2'
            cmdl = ['sh', '-c', cmdl]
            rc = testbed.execute(what, cmdl, script=None, kind='install')
            if rc:
                badpkg('installation of %s failed, exit code %d'
                       % (pkg, rc))

        self._debug('publish done')

#---------- processing of sources (building)


def source_rules_command(script, what, which, cwd,
                         results_lines=0, xargs=[]):
    if opts.debuglevel >= 1:
        script = ['exec 3>&1 >&2', 'set -x'] + script
    else:
        script = ['exec 3>&1 >&2'] + script
    script = '\n'.join(script)
    so = TempTestbedPath(testbed, '%s-%s-results' % (what, which))
    rc = testbed.execute('%s-%s' % (what, which),
                         ['sh', '-ec', script] + xargs, script=script,
                         so=so.tb, cwd=cwd, kind='build')
    so.copyup()
    with open(so.host) as f:
        results = f.read().rstrip('\n')
    if len(results):
        results = results.split('\n')
    else:
        results = []
    if rc:
        badpkg('rules %s failed with exit code %d' % (which, rc))
    if results_lines is not None and len(results) != results_lines:
        badpkg('got %d lines of results from %s where %d expected'
               % (len(results), which, results_lines))
    if results_lines == 1:
        return results[0]
    return results


def build_source(act):
    act.blame = 'arg:' + act.arg
    testbed.blame(act.blame)
    testbed.prepare1([])

    what = act.what
    act.binaries = []

    def debug_b(m):
        debug('* <%s:%s> %s' % (act.kind, act.what, m))

    if act.kind == 'dsc':
        dsc = act.arg
        dsc_tb = os.path.join(testbed.scratch, what, os.path.basename(dsc))

        # copy .dsc file itself
        TestbedPath(testbed, dsc, dsc_tb).copydown()

        # parse source file parts from the .dsc and copy them to testbed
        in_files = False
        fre = re.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
        with open(dsc) as f:
            for l in f:
                l = l.rstrip('\n')
                if l.startswith('Files:'):
                    in_files = True
                    continue
                elif l.startswith('#'):
                    pass
                elif not l.startswith(' '):
                    in_files = False
                    if l.startswith('Source:'):
                        act.blame = 'dsc:' + l[7:].strip()
                        testbed.blame(act.blame)
                if not in_files:
                    continue

                m = fre.match(l)
                if not m:
                    badpkg(".dsc contains unparseable line"
                           " in Files: `%s'" % l)
                leaf = m.groups(0)[0]

                part = TestbedPath(
                    testbed,
                    os.path.join(os.path.dirname(dsc), leaf),
                    os.path.join(testbed.scratch, what, leaf))
                part.copydown()

    if act.kind == 'ubtree':
        dsc = os.path.join(tmp, what + '-fakedsc')
        with open(dsc, 'w') as f_dsc:
            with open(os.path.join(act.arg, 'debian/control')) as f_control:
                for l in f_control:
                    if l == '\n':
                        break
                    f_dsc.write(l)
            f_dsc.write('Binary: none-so-this-is-not-a-package-name\n')
        atexit.register(lambda f: os.path.exists(f) and os.unlink(f), dsc)

    if act.kind in ['dsc', 'apt']:
        testbed.prepare2([])
        testbed.satisfy_dependencies_string('dpkg-dev',
                                            'install dpkg-dev')

    # we only use the testbed path from this
    work = os.path.join(testbed.scratch, what + '-build')

    tmpdir = work + '/tmpdir'
    tmpdir_script = [
        'TMPDIR="$1"',
        'rm -rf -- "$TMPDIR"',
        'export TMPDIR',
        opts.user_wrap('mkdir -m 1777 -- "$TMPDIR"'),
    ]

    if act.kind == 'ubtree':
        spec = '%s/real-tree' % work
        create_command = '''
    rm -rf "$spec"
    mkdir "$spec"
    cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
'''
        # copy unbuilt tree into testbed
        ubtree = TestbedPath(testbed, act.arg,
                             os.path.join(testbed.scratch, act.what + '-ubtree'))
        ubtree.copydown()
        initcwd = ubtree.tb

    if act.kind == 'dsc':
        spec = dsc_tb
        create_command = 'dpkg-source -x $spec\n'
        initcwd = work

    if act.kind == 'apt':
        spec = act.arg
        create_command = 'apt-get source $spec\n'
        initcwd = work

    script = [
        'spec="$2"',
        'origpwd=' + initcwd,
        'mkdir -p ' + work,
        'cd ' + work
    ]

    if opts.user:
        script += (['chown ' + opts.user + ' . ..'] +
                   tmpdir_script +
                   ['spec="$spec" origpwd="$origpwd" '
                    + opts.user_wrap(create_command)])
    else:
        script += (tmpdir_script +
                   [create_command])

    script += [
        'cd [a-z0-9]*-*/.',
        'pwd >&3',
        'set +e; test -f debian/tests/control; echo $? >&3'
    ]
    (result_pwd, control_test_rc) = source_rules_command(
        script, what, 'extract',
        cwd=None, results_lines=2, xargs=['x', tmpdir, spec])

    filter = act.ah['dsc_filter']

    if control_test_rc == '1':
        act.missing_tests_control = True

    # For optional builds:
    #
    # We might need to build the package because:
    #   - we want its binaries (filter isn't _ and at least one of the
    #	deb_... isn't ignore)
    #   - the test control file says so
    #       (assuming we have any tests)

    class NeedBuildException:
        pass

    def build_needed(m):
        debug_b('build needed for %s' % m)
        raise NeedBuildException()

    try:
        if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
                              act.ah['deb_fortests'] != 'ignore'):
            build_needed('binaries')

        # get test control file from testbed
        test_control = TestbedPath(testbed,
                                   os.path.join(tmp, what + '-testcontrol'),
                                   os.path.join(result_pwd, 'debian/tests/control'), False)
        test_control.copyup()
        stanzas = read_control(act, test_control.host)
        os.unlink(test_control.host)
        for stanza in stanzas:
            for t in stanza[' tests']:
                if 'build-needed' in t.restriction_names:
                    build_needed('test %s' % t.tname)

        debug_b('build not needed')
        built = False

    except NeedBuildException:
        testbed.needs_reset()

        if act.kind not in ['dsc', 'apt']:
            testbed.prepare2([])

        if act.kind == 'apt':
            # we need to get the downloaded debian/control from the testbed, so
            # that we can avoid calling "apt-get build-dep" and thus
            # introducing a second mechanism for installing build deps
            pkg_control = TestbedPath(testbed,
                                      os.path.join(tmp, what + '-control'),
                                      os.path.join(result_pwd, 'debian/control'), False)
            pkg_control.copyup()
            dsc = os.path.join(tmp, what + '-fakedsc')
            with open(dsc, 'w') as f_dsc:
                with open(pkg_control.host) as f_control:
                    for l in f_control:
                        if l == '\n':
                            break
                        f_dsc.write(l)
                f_dsc.write('Binary: none-so-this-is-not-a-package-name\n')
            atexit.register(lambda f: os.path.exists(f) and os.unlink(f), dsc)

        testbed.satisfy_dependencies_string(', '.join(build_essential),
                                            'install build-essential')
        testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')

        script = tmpdir_script + [
            'cd "$2"',
            'dpkg-checkbuilddeps',
            opts.user_wrap('debian/rules build'),
        ]
        source_rules_command(script, what, 'build',
                             cwd=initcwd, xargs=['x', tmpdir, result_pwd])

        if os.path.dirname(result_pwd) != work:
            badpkg("results dir `%s' is not in expected parent"
                   " dir `%s'" % (result_pwd, work))

        built = True

    act.tests_tree = TestbedPath(
        testbed,
        os.path.join(tmp, what + '-tests-tree'),
        os.path.join(work, os.path.basename(result_pwd)),
        is_dir=True)
    # copy tests from testbed to hosts, we need that for parsing control files
    # and some runners like LXC run a fresh testbed after build for the tests
    act.tests_tree.copyup()
    atexit.register(rmtree, 'tests-tree', act.tests_tree.host)

    if not built:
        act.blamed = []
        return

    act.blamed = copy.copy(testbed.blamed)

    debug_b('filter=%s' % filter)
    if filter != '_':
        script = tmpdir_script + [
            'cd ' + work + '/[a-z0-9]*-*/.',
            opts.user_wrap(opts.gainroot + ' debian/rules binary'),
            'cd ..',
            'echo *.deb >&3',
        ]
        result_debs = source_rules_command(script, what,
                                           'binary', work,
                                           results_lines=1,
                                           xargs=['x', tmpdir])
        if result_debs == '*.deb':
            debs = []
        else:
            debs = result_debs.split(' ')
        debug_b('debs=' + repr(debs))

        # determine built debs and copy them from testbed
        deb_re = re.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
        for deb in debs:
            m = deb_re.match(deb)
            if not m:
                badpkg("badly-named binary `%s'" % deb)
            pkg = m.groups()[0]
            debug_b(' deb=%s, pkg=%s' % (deb, pkg))
            for pat in filter.split(','):
                debug_b('  pat=%s' % pat)
                if not fnmatch.fnmatchcase(pkg, pat):
                    debug_b('   no match')
                    continue
                deb_what = pkg + '_' + what + '.deb'
                deb_path = TestbedPath(testbed,
                                       os.path.join(tmp, deb_what),
                                       os.path.join(work, deb),
                                       False)
                debug_b('  deb_what=%s, deb_path=%s' %
                        (deb_what, str(deb_path)))
                deb_path.copyup()
                binaries.register(act, pkg, deb_path.host,
                                  'forbuilds', testbed.blamed)
                act.binaries.append((pkg, deb_path.host))
                break
        debug_b('all done.')

#---------- main processing loop and main program


def process_actions():
    global binaries

    def debug_a1(m):
        debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ ' + m)

    def debug_a2(m):
        debug('@@@@@@@@@@@@@@@@@@@@ ' + m)

    def debug_a3(m):
        debug('@@@@@@@@@@ ' + m)

    testbed.open()

    binaries = Binaries(testbed)

    binaries.reset()

    debug_a1('builds ...')
    for act in opts.actions:
        debug_a2('%s %s' % (act.kind, act.what))

        if act.kind == 'deb':
            testbed.blame('arg:' + act.arg)
            determine_package(act)
            testbed.blame('deb:' + act.pkg)
            binaries.register(act, act.pkg, act.arg,
                              'forbuilds', testbed.blamed)
        if act.kind in ['dsc', 'ubtree', 'apt']:
            build_source(act)
        if act.kind == 'tree':
            act.binaries = []
        if act.kind == 'instantiate':
            pass

    debug_a1('builds done.')

    binaries.reset()

    debug_a1('tests ...')
    for act in opts.actions:
        debug_a2('test %s %s' % (act.kind, act.what))

        if act.kind == 'deb':
            binaries.register(act, act.pkg, act.arg, 'fortests',
                              ['deb:' + act.pkg])
        if act.kind == 'dsc' or act.kind == 'ubtree':
            for (pkg, path) in act.binaries:
                binaries.register(act, pkg, path, 'fortests',
                                  act.blamed)
        if act.kind in ['dsc', 'apt']:
            if act.ah['dsc_tests']:
                debug_a3('read control ...')
                stanzas = read_control(
                    act, os.path.join(act.tests_tree.host, 'debian/tests/control'))
                testbed.blamed += act.blamed
                debug_a3('run_tests ...')
                run_tests(stanzas, act.tests_tree)
        if act.kind == 'tree' or act.kind == 'ubtree':
            testbed.blame('arg:' + act.arg)
            stanzas = read_control(
                act, os.path.join(act.arg, 'debian/tests/control'))
            debug_a3('run_tests ...')
            if act.kind == 'ubtree':
                run_tests(stanzas, act.tests_tree)
            else:
                tree = TestbedPath(testbed, act.arg,
                                   os.path.join(testbed.scratch, act.what + '-tree'))
                run_tests(stanzas, tree)
        if act.kind == 'instantiate':
            testbed.prepare([])
        testbed.needs_reset()
    debug_a1('tests done.')


def main():
    global testbed
    try:
        parse_args()
    except SystemExit:
        # optparser exits with error 2 by default, but we have a different
        # meaning for that already
        sys.exit(20)

    try:
        setup_trace()
        testbed = Testbed()
        testbed.start()
        finalise_options()
        process_actions()
    except:
        ec = print_exception(sys.exc_info(), '')
        cleanup()
        sys.exit(ec)
    cleanup()
    sys.exit(errorcode)

main()
