One Hat Cyber Team

View File Name : bugzilla

#!/usr/bin/python2 -tt
#
# bugzilla - a commandline frontend for the python bugzilla module
#
# Copyright (C) 2007, 2008, 2009, 2010, 2011 Red Hat Inc.
# Author: Will Woods <wwoods@redhat.com>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.

from __future__ import print_function

import locale
from logging import DEBUG, INFO, WARN, StreamHandler, Formatter
import optparse
import os
import re
import socket
import sys
import tempfile

if hasattr(sys.version_info, "major") and sys.version_info.major >= 3:
    # pylint: disable=F0401,W0622
    from xmlrpc.client import Fault, ProtocolError
    basestring = (str, bytes)
else:
    from xmlrpclib import Fault, ProtocolError

import requests.exceptions

import bugzilla

default_bz = 'https://bugzilla.redhat.com/xmlrpc.cgi'

_is_unittest = bool(os.getenv("__BUGZILLA_UNITTEST"))
cmdlist = ['login', 'new', 'query', 'modify', 'attach', 'info']
format_field_re = re.compile("%{([a-z0-9_]+)(?::([^}]*))?}")

log = bugzilla.log
handler = StreamHandler(sys.stderr)
handler.setFormatter(Formatter(
    "[%(asctime)s] %(levelname)s (%(module)s:%(lineno)d) %(message)s",
    "%H:%M:%S"))
log.addHandler(handler)


################
# Util helpers #
################

def to_encoding(ustring):
    string = ''
    if isinstance(ustring, basestring):
        string = ustring
    elif ustring is not None:
        string = str(ustring)

    if hasattr(sys.version_info, "major") and sys.version_info.major >= 3:
        return string
    else:
        preferred = locale.getpreferredencoding()
        if "PYTHON_BUGZILLA_TEST_SUITE" in os.environ:
            preferred = "UTF-8"
        return string.encode(preferred, 'replace')


def open_without_clobber(name, *args):
    '''Try to open the given file with the given mode; if that filename exists,
    try "name.1", "name.2", etc. until we find an unused filename.'''
    fd = None
    count = 1
    orig_name = name
    while fd is None:
        try:
            fd = os.open(name, os.O_CREAT | os.O_EXCL, 0o666)
        except OSError:
            err = sys.exc_info()[1]
            if err.errno == os.errno.EEXIST:
                name = "%s.%i" % (orig_name, count)
                count += 1
            else:
                raise IOError(err.errno, err.strerror, err.filename)
    fobj = open(name, *args)
    if fd != fobj.fileno():
        os.close(fd)
    return fobj


##################
# Option parsing #
##################

def setup_parser():
    u = ("%%prog [global options] COMMAND [command-options]"
         "\n\nCommands: %s" % ', '.join(sorted(cmdlist)))
    p = optparse.OptionParser(usage=u, version=bugzilla.__version__)
    p.disable_interspersed_args()
    p.epilog = 'Try "bugzilla COMMAND --help" for command-specific help.'

    # General bugzilla connection options
    p.add_option('--bugzilla', default=default_bz,
            help="bugzilla XMLRPC URI. default: %s" % default_bz)
    p.add_option("--nosslverify", dest="sslverify",
                 action="store_false", default=True,
                 help="Don't error on invalid bugzilla SSL certificate")

    p.add_option('--user', help="Optional bugzilla login username")
    p.add_option('--password', help="Optional bugzilla login password")
    p.add_option('--login', action="store_true",
        help='Run interactive "login" before performing the '
             'specified command.')
    p.add_option('--ensure-logged-in', action="store_true",
        help="Raise an error if we aren't logged in to bugzilla. "
             "Consider using this if you are depending on "
             "cached credentials, to ensure that when they expire the "
             "tool errors, rather than subtly change output.")
    p.add_option('--no-cache-credentials',
        action='store_false', default=True, dest='cache_credentials',
        help="Don't save any bugzilla cookies or tokens to disk, and "
             "don't use any pre-existing credentials.")

    p.add_option('--cookiefile', default=None,
            help="cookie file to use for bugzilla authentication")
    p.add_option('--tokenfile', default=None,
            help="token file to use for bugzilla authentication")

    p.add_option('--verbose', action='store_true',
            help="give more info about what's going on")
    p.add_option('--debug', action='store_true',
            help="output bunches of debugging info")

    # Generate the man page, dump to stdout. Not for end users
    p.add_option('--generate-man', action='store_true',
            help=optparse.SUPPRESS_HELP)

    # Allow user to specify BZClass to initialize. Kinda weird for the
    # CLI, I'd rather people file bugs about this so we can fix our detection.
    # So hide it from the help output but keep it for back compat
    p.add_option('--bztype', default='auto', help=optparse.SUPPRESS_HELP)

    return p


def setup_action_parser(action):
    p = optparse.OptionParser(usage="%%prog %s [options]" % action)

    if action == 'new':
        p.set_description("Create a new bug report.")
        p.add_option('-p', '--product', help="REQUIRED: product name")
        p.add_option('-v', '--version', help="REQUIRED: product version")
        p.add_option('-c', '--component', help="REQUIRED: component name")
        p.add_option("--sub-component", help="Optional sub component name")
        p.add_option('-s', '--short_desc', '--summary', dest='summary',
                help="REQUIRED: bug summary")
        p.add_option('-l', '--comment', dest='description',
                help="initial bug comment")
        p.add_option('-o', '--os', help="Operating system")
        p.add_option('-a', '--arch', help="Arch this bug occurs on")
        p.add_option('--severity', help="Bug severity")
        p.add_option('--priority', help="Bug priority")
        p.add_option('-u', '--url', help="URL for further bug info")
        p.add_option('--cc',
                metavar='CC[, CC, ...]', action="append",
                help="add emails to initial CC list")
        p.add_option('--blocked',
                metavar='BUGID[, BUGID, ...]', action="append",
                help="add bug_ids blocked by this bug")
        p.add_option('--dependson',
                metavar='BUGID[, BUGID, ...]', action="append",
                help="add bug_ids that this bug depends on")
        p.add_option('--groups',
                metavar='GROUP[, GROUP, ...]', action="append",
                help="add groups to which bug is visible")
        p.add_option('--assigned_to',
                help='Assign bugzilla to specified email address')
        p.add_option('--qa_contact',
                help='Set QA contact to specified email address')
        p.add_option('--keywords',
                metavar='KEYWORD[, KEYWORD, ...]', action="append",
                help="Set specified keywords on new bugzilla")
        p.add_option('--alias',
                help='An alias (name) for the bug (must be unique)')

    elif action == 'query':
        p.set_description("List bug reports that match the given criteria.")
        # General bug metadata
        p.add_option('-b', '--bug_id', default=None,
                help="specify individual bugs by IDs, separated with commas")
        p.add_option('-p', '--product',
                help="product name, comma-separated "
                     "(list with 'bugzilla info --products')")
        p.add_option('-v', '--version',
                help="product version, comma-separated")
        p.add_option('-c', '--component',
                help="component name(s), comma-separated "
                     "(list with 'bugzilla info --components PRODUCT')")
        p.add_option("--sub-component", action="append",
            help="Sub component. Can be specified multiple times")
        p.add_option('--components_file', default=None,
                help="list of component names from a file, one component "
                     "per line (list with 'bugzilla info -c PRODUCT')")
        p.add_option('-l', '--long_desc',
                help="search inside bug comments")
        p.add_option('-m', '--target_milestone',
                help="search for a target milestone")
        p.add_option('-s', '--short_desc',
                help="search bug summaries")
        p.add_option('-t', '--bug_status', default="ALL",
                help="comma-separated list of bug statuses to accept "
                     "[Default:all]")
        p.add_option('-x', '--bug_severity', '--severity',
                help="search severities, comma-separated")
        p.add_option('-z', '--priority',
                help="search priorities, comma-separated")

        # Email
        malg = optparse.OptionGroup(p, "Email Options")
        malg.add_option('-E', '--emailtype', default="substring",
                help="Email: specify searching option for emails, "
                     "ie. substring, notsubstring, exact, ... "
                     "[Default: substring]")
        malg.add_option('-o', '--cc',
                help="Email: search cc lists for given address")
        malg.add_option('-r', '--reporter',
                help="Email: search reporter email for given address")
        malg.add_option('-a', '--assigned_to',
                help="Email: search for bugs assigned to this address")
        malg.add_option('-q', '--qa_contact',
                help="Email: search for bugs which have QA Contact "
                     "assigned to this address")
        p.add_option_group(malg)

        # Strings
        stro = optparse.OptionGroup(p, "String search options")
        stro.add_option('-u', '--url',
                help="search keywords field for given url")
        stro.add_option('-U', '--url_type',
                help="specify searching option for urls, "
                     "ie. anywords, allwords, nowords")
        stro.add_option('-k', '--keywords',
                help="search keywords field for specified words")
        stro.add_option('-K', '--keywords_type',
                help="specify searching option for keywords, "
                     "ie. anywords, allwords, nowords")
        stro.add_option('-w', '--status_whiteboard',
                help="search Status Whiteboard field for specified words")
        stro.add_option('-W', '--status_whiteboard_type',
                help="specify searching option for Status Whiteboard, "
                     "ie. anywords, allwords, nowords")
        stro.add_option("--tags", help="Search bug 'tags' field")
        p.add_option_group(stro)

        # Precomposed queries
        p.add_option('--from-url',
            help="Use the query given by a query.cgi URL. (Use quotes!)")
        p.add_option('--quicksearch',
            help="Search using bugzilla's quicksearch functionality.")
        p.add_option('--savedsearch',
            help="Name of a bugzilla saved search. If you don't own this "
                "saved search, you must passed --savedsearch_sharer_id.")
        p.add_option('--savedsearch-sharer-id',
            help="Owner ID of the --savedsearch. You can get this ID from "
                "the URL bugzilla generates when running the saved search "
                "from the web UI.")

        # Boolean Charts
        bgrp = optparse.OptionGroup(p, "Boolean options")
        bgrp.add_option('-B', '--booleantype', default="substring",
                help="specify searching option for booleans, ie. substring, "
                     "notsubstring, exact, ... [Default: substring]")
        bgrp.add_option('--boolean_query', action="append",
                help="Create your own query. Format: "
                     "BooleanName-Condition-Parameter &/| ... . ie, "
                     "keywords-substring-Partner & "
                     "keywords-notsubstring-OtherQA")
        bgrp.add_option('--blocked', action="append",
                help="Search for bugs that block this bug ID")
        bgrp.add_option('--dependson', action="append",
                help="Search for bugs that depend on this bug ID")
        bgrp.add_option('--flag', action='append',
                help="Search for bugs that have certain "
                     "flag states present. Ex --flags=dev_ack+")
        bgrp.add_option('--qa_whiteboard', action="append",
                help="search for bugs that have certain QA "
                     "Whiteboard text present")
        bgrp.add_option('--devel_whiteboard', action="append",
                help="search for bugs that have certain "
                     "Devel Whiteboard text present")
        bgrp.add_option('--alias', action="append",
                help="search for bugs that have the provided alias")
        bgrp.add_option('--fixed_in', action="append",
                help="search Status Whiteboard field for specified words")
        p.add_option_group(bgrp)

    elif action == 'info':
        p.set_description("Get information about the bugzilla server.")
        p.add_option('-p', '--products', action='store_true',
                help='Get a list of products')
        p.add_option('-c', '--components', metavar="PRODUCT",
                help='List the components in the given product')
        p.add_option('-o', '--component_owners', metavar="PRODUCT",
                help='List components (and their owners)')
        p.add_option('-v', '--versions', metavar="VERSION",
                help='List the versions for the given product')

    elif action == 'modify':
        p.set_usage("%prog modify [options] BUGID [BUGID...]")
        p.set_description("Modify one or more bugs.")

        bgrp = optparse.OptionGroup(p, "Bug details")
        bgrp.add_option('--product',
                help="Reassign bug to different product")
        bgrp.add_option('-c', '--component',
                help="Reassign bug to different component")
        bgrp.add_option("--sub-component",
            help="Reassign bug to different sub-component (rhbz extension)")
        bgrp.add_option("-v", '--version',
                help="Reassign bug to different version")
        bgrp.add_option('-o', '--os',
                help="Change operating system this bug occurs on")
        bgrp.add_option('-a', '--arch',
                help="Change arch this bug occurs on")
        bgrp.add_option('-u', '--url',
                help="URL for further bug info")
        bgrp.add_option('--alias',
                help='An alias (name) for the bug (must be unique)')
        p.add_option_group(bgrp)

        sgrp = optparse.OptionGroup(p, "Bug status options")
        sgrp.add_option('-s', '--status',
                help='Change status of bug')
        sgrp.add_option('-k', '--close', metavar="RESOLUTION",
                help='Close with the given resolution')
        sgrp.add_option('-d', '--dupeid', metavar="ORIGINAL",
                help='ID of original bug (implies -k DUPLICATE)')
        sgrp.add_option('-F', '--fixed_in', metavar="VERSION",
                help='"Fixed in version" field')
        p.add_option_group(sgrp)

        cgrp = optparse.OptionGroup(p, "Comment options")
        cgrp.add_option('-l', '--comment',
                help='Add a comment')
        cgrp.add_option('-p', '--private', action='store_true', default=False,
                help='Mark new comment as private')
        cgrp.add_option("--summary", help="Change bug summary")
        p.add_option_group(cgrp)

        egrp = optparse.OptionGroup(p, "Contact options")
        egrp.add_option('--assignee',
                help='Assign bugzilla to assignee')
        egrp.add_option('--cc', action='append', metavar="EMAIL",
                help='Alter CC list. EMAIL appends, -EMAIL removes.')
        egrp.add_option('--qa_contact',
                help='Change QA contact')
        egrp.add_option('--reset-assignee', action="store_true",
                help='Reset assignee to component default')
        egrp.add_option('--reset-qa-contact', action="store_true",
                help='Reset QA contact to component default')
        egrp.add_option('--groups', metavar='GROUP[, GROUP, ...]',
                action="append",
                help="add groups to which bug is visible")
        p.add_option_group(egrp)

        tgrp = optparse.OptionGroup(p, "Tracking options")
        tgrp.add_option('-f', '--flag', action='append',
                help='Update bugzilla flags with requested type, '
                     'ie fedora-cvs?, or needinfoX to clear '
                     '(Use a new option for each flag)')
        tgrp.add_option('--severity', help="Change bug severity")
        tgrp.add_option('--priority', help="Change bug priority")
        tgrp.add_option('--target_milestone', help="Set target milestone")
        tgrp.add_option('--target_release', help="Set target release")
        tgrp.add_option('--blocked',
                metavar='BUGID[, BUGID, ...]', action="append",
                help="Add bug_ids blocked by this bug. BUGID appends, "
                     "-BUGID removes, =BUGID overwrites")
        tgrp.add_option('--dependson', metavar='BUGID[, BUGID, ...]',
                action="append",
                help=('Alter depends_on list.  BUGID appends, '
                     '-BUGID removes, =BUGID overwrites'))
        tgrp.add_option('--keywords',
                metavar='KEYWORD', action="append",
                help="Alter bug keywords list. KEYWORD appends, "
                     "-KEYWORD removes, =KEYWORD overwrites")
        tgrp.add_option("", "--whiteboard", metavar="TEXT", action="append",
                help='Alter status whiteboard text. '
                      'TEXT appends, -TEXT removes, =TEXT overwrites')
        tgrp.add_option("--devel_whiteboard",
                metavar="TEXT", action="append",
                help='Alter devel whiteboard text. '
                      'TEXT appends, -TEXT removes, =TEXT overwrites')
        tgrp.add_option("--internal_whiteboard",
                metavar="TEXT", action="append",
                help='Alter internal whiteboard text. '
                      'TEXT appends, -TEXT removes, =TEXT overwrites')
        tgrp.add_option("--qa_whiteboard",
                metavar="TEXT", action="append",
                help='Alter QA whiteboard. '
                      'TEXT appends, -TEXT removes, =TEXT overwrites')
        tgrp.add_option("--tags", metavar="TEXT", action="append",
                help="Alter bug 'tags' field"
                     "TEXT appends, -TEXT removes")
        p.add_option_group(tgrp)


    elif action == 'attach':
        p.set_usage('''
  %prog attach --file=FILE --desc=DESC [--type=TYPE] BUGID [BUGID...]
  %prog attach --get=ATTACHID --getall=BUGID [...]
  %prog attach --type=TYPE BUGID [BUGID...]''')
        p.set_description("Attach files or download attachments.")
        p.add_option('-f', '--file', metavar="FILENAME",
                help='File to attach, or filename for data provided on stdin')
        p.add_option('-d', '--description', metavar="DESCRIPTION", dest='desc',
                help="A short description of the file being attached")
        p.add_option('-t', '--type', metavar="MIMETYPE",
                help="Mime-type for the file being attached")
        p.add_option('-g', '--get', metavar="ATTACHID", action="append",
                default=[], help="Download the attachment with the given ID")
        p.add_option("--getall", "--get-all", metavar="BUGID", action="append",
                default=[], help="Download all attachments on the given bug")

    elif action == 'login':
        p.set_usage('%prog login [username [password]]')
        p.set_description(
            "Log into bugzilla and save a login cookie or token.")

    if action in ['new', 'query']:
        outg = optparse.OptionGroup(p, "Output format options")
        outg.add_option('-f', '--full', action='store_const', dest='output',
                const='full', default='normal',
                help="output detailed bug info")
        outg.add_option('-i', '--ids', action='store_const', dest='output',
                const='ids', help="output only bug IDs")
        outg.add_option('-e', '--extra', action='store_const',
                dest='output', const='extra',
                help="output additional bug information "
                     "(keywords, Whiteboards, etc.)")
        outg.add_option('--oneline', action='store_const', dest='output',
                const='oneline',
                help="one line summary of the bug (useful for scripts)")
        outg.add_option('--raw', action='store_const', dest='output',
                const='raw', help="raw output of the bugzilla contents")
        outg.add_option('--outputformat',
                help="Print output in the form given. "
                     "You can use RPM-style tags that match bug "
                     "fields, e.g.: '%{id}: %{summary}'. See the man page "
                     "section 'OUTPUT FORMAT' for more details.")
        p.add_option_group(outg)

    if action in ['new', 'query', 'modify']:
        message = {
            'new': 'Set a specified field.',
            'query': 'Query a specified field.',
            'modify': 'Modify a specified field.'
        }.get(action)
        p.add_option('--field', help="%s FIELD is expected to be \
                     the raw name used by the bugzilla instance. No safety \
                     checks are perfomed when using this option." % (message),
                     action="append", type="str", dest="fields",
                     metavar="FIELD=VALUE")


    # Used by unit tests, not for end user consumption
    if action in ['new', 'query', 'modify']:
        p.add_option('--test-return-result', action="store_true",
                     help=optparse.SUPPRESS_HELP)

    return p


####################
# Command routines #
####################

def generate_man_page():
    from logilab.common.optik_ext import ManHelpFormatter
    import datetime

    today = datetime.date.today()
    datestr = today.strftime("%B %d, %Y")

    # pylint: disable=W1401
    # Anomalous backslash in string, man format confuses pylint
    manpage = \
'''.TH bugzilla 1  "%s" "version %s" "User Commands"
.SH NAME
bugzilla \- command-line interface to Bugzilla over XML-RPC
.SH SYNOPSIS
.B bugzilla
[\\fIoptions\\fR] [\\fIcommand\\fR] [\\fIcommand-options\\fR]
.SH DESCRIPTION
.PP
.BR bugzilla
is a command-line utility that allows access to the XML-RPC interface provided
by Bugzilla.
.PP
\\fIcommand\\fP is one of:
.br
.I \\fR * login - log into the given bugzilla instance
.br
.I \\fR * new - create a new bug
.br
.I \\fR * query - search for bugs matching given criteria
.br
.I \\fR * modify - modify existing bugs
.br
.I \\fR * attach - attach files to existing bugs, or get attachments
.br
.I \\fR * info - get info about the given bugzilla instance
''' % (datestr, bugzilla.__version__)

    manformatter = ManHelpFormatter()
    parser = setup_parser()
    parser.formatter = manformatter

    opt_section = parser.format_option_help()
    manpage += opt_section.replace("OPTIONS", "GLOBAL OPTIONS")
    for action in cmdlist:
        action_parser = setup_action_parser(action)
        action_parser.remove_option("--help")
        action_parser.formatter = manformatter
        opt_section = action_parser.format_option_help()
        manpage += opt_section.replace("OPTIONS",
                                       '\[oq]%s\[cq] OPTIONS' % action.upper())
    manpage += \
'''.SH OUTPUT FORMAT

The output of the bugzilla tool should NEVER BE PARSED unless you are using a
custom --outputformat. For everything else, just don't parse it, the formats
are not stable and are subject to change.

--outputformat allows printing arbitrary bug data in a user preferred format.
For example, to print a returned bug ID, component, and product, separated
with ::, do:

--outputformat "%{id}::%{component}::%{product}"

The fields (like 'id', 'component', etc.) are the names of the values returned
by bugzilla's XMLRPC interface. To see a list of all fields, check the API
documentation in the 'SEE ALSO' section. Alternatively, run a 'bugzilla
--debug query ...' and look at the key names returned in the query results.
Also, in most cases, using the name of the associated command line switch
should work, like --bug_status becomes %{bug_status}, etc.

.SH AUTHENTICATION COOKIES AND TOKENS

Older bugzilla instances use cookie-based authentication, and
bugzilla.redhat.com uses a non-cookie token system.

When you log into bugzilla with the "login" subcommand or the "--login"
argument, we cache the cookie in ~/.bugzillacookies. If you are using
bugzilla.redhat.com, we also cache the token in ~/.bugzillatoken.

To perform an authenticated bugzilla command on a new machine, run a one time
"bugzilla login" to cache credentials before running the desired command. You
can also run "bugzilla --login" and the login process will be initiated before
invoking the command.

Additionally, the --no-cache-credentials option will tell the bugzilla tool to
_not_ save any credentials to ~/.bugzillacookies or ~/.bugzillatoken.

.SH EXAMPLES
.PP
.RS 0
bugzilla query --bug_id 62037

bugzilla query --version 15 --component python-bugzilla

# All boolean options can be formatted like this
.br
bugzilla query --blocked "123456 | 224466"

bugzilla login

bugzilla new -p Fedora -v rawhide -c python-bugzilla \\\\
         --summary "python-bugzilla causes headaches" \\\\
         --comment "python-bugzilla made my brain hurt when I used it."

bugzilla attach --file ~/Pictures/cam1.jpg --desc "me, in pain" $BUGID

bugzilla attach --getall $BUGID

bugzilla modify --close NOTABUG --comment "Actually, you're hungover." $BUGID


.SH EXIT STATUS
.BR bugzilla
normally returns 0 if the requested command was successful.
Otherwise, exit status is 1 if
.BR bugzilla
is interrupted by the user (or a login attempt fails), 2 if a
socket error occurs (e.g. TCP connection timeout), and 3 if the server returns
an XML-RPC fault.
.SH BUGS
Please report any bugs as github issues at
.br
https://github.com/python-bugzilla/python-bugzilla
.br
to the mailing list at
.br
https://fedorahosted.org/mailman/listinfo/python-bugzilla
.SH SEE ALSO
.nf
http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html
https://bugzilla.redhat.com/docs/en/html/api/Bugzilla/WebService/Bug.html'''

    print(manpage)


def _merge_field_opts(query, opt, parser):
    # Add any custom fields if specified
    if opt.fields is None:
        return

    for f in opt.fields:
        try:
            f, v = f.split('=', 1)
            query[f] = v
        except:
            parser.error("Invalid field argument provided: %s" % (f))


def _do_query(bz, opt, parser):
    # Construct the query from the list of queryable options
    q = dict()
    # Parse preconstructed queries.
    u = getattr(opt, 'from_url', None)
    if u:
        q = bz.url_to_query(u)

    if opt.components_file:
        # Components slurped in from file (one component per line)
        # This can be made more robust
        clist = []
        f = open(opt.components_file, 'r')
        for line in f.readlines():
            line = line.rstrip("\n")
            clist.append(line)
        opt.component = clist

    if opt.bug_status:
        val = opt.bug_status
        stat = val
        if val == 'ALL':
            # leaving this out should return bugs of any status
            stat = None
        elif val == 'DEV':
            # Alias for all development bug statuses
            stat = ['NEW', 'ASSIGNED', 'NEEDINFO', 'ON_DEV',
                'MODIFIED', 'POST', 'REOPENED']
        elif val == 'QE':
            # Alias for all QE relevant bug statuses
            stat = ['ASSIGNED', 'ON_QA', 'FAILS_QA', 'PASSES_QA']
        elif val == 'EOL':
            # Alias for EndOfLife bug statuses
            stat = ['VERIFIED', 'RELEASE_PENDING', 'CLOSED']
        elif val == 'OPEN':
            # non-Closed statuses
            stat = ['NEW', 'ASSIGNED', 'MODIFIED', 'ON_DEV', 'ON_QA',
                'VERIFIED', 'RELEASE_PENDING', 'POST']
        opt.bug_status = stat

    # Convert all comma separated list parameters to actual lists,
    # which is what bugzilla wants
    # According to bugzilla docs, any parameter can be a list, but
    # let's only do this for options we explicitly mention can be
    # comma separated.
    for optname in ["bug_severity", "bug_id", "bug_status", "component",
                    "priority", "product", "version"]:
        val = getattr(opt, optname, None)
        if not isinstance(val, str):
            continue
        setattr(opt, optname, val.split(","))

    include_fields = None
    if opt.output == 'raw':
        # 'raw' always does a getbug() call anyways, so just ask for ID back
        include_fields = ['id']

    elif opt.outputformat:
        include_fields = []
        for fieldname, rest in format_field_re.findall(opt.outputformat):
            if fieldname == "whiteboard" and rest:
                fieldname = rest + "_" + fieldname
            elif fieldname == "flag":
                fieldname = "flags"
            elif fieldname == "cve":
                fieldname = ["keywords", "blocks"]
            elif fieldname == "__unicode__":
                # Needs to be in sync with bug.__unicode__
                fieldname = ["id", "status", "assigned_to", "summary"]

            flist = isinstance(fieldname, list) and fieldname or [fieldname]
            for f in flist:
                if f not in include_fields:
                    include_fields.append(f)

    if include_fields is not None:
        include_fields.sort()

    built_query = bz.build_query(
        product=getattr(opt, "product", None),
        component=getattr(opt, "component", None),
        sub_component=getattr(opt, "sub_component", None),
        version=getattr(opt, "version", None),
        reporter=getattr(opt, "reporter", None),
        bug_id=getattr(opt, "bug_id", None),
        short_desc=getattr(opt, "short_desc", None),
        long_desc=getattr(opt, "long_desc", None),
        cc=getattr(opt, "cc", None),
        assigned_to=getattr(opt, "assigned_to", None),
        qa_contact=getattr(opt, "qa_contact", None),
        status=getattr(opt, "bug_status", None),
        blocked=getattr(opt, "blocked", None),
        dependson=getattr(opt, "dependson", None),
        keywords=getattr(opt, "keywords", None),
        keywords_type=getattr(opt, "keywords_type", None),
        url=getattr(opt, "url", None),
        url_type=getattr(opt, "url_type", None),
        status_whiteboard=getattr(opt, "status_whiteboard", None),
        status_whiteboard_type=getattr(opt, "status_whiteboard_type", None),
        fixed_in=getattr(opt, "fixed_in", None),
        fixed_in_type=getattr(opt, "fixed_in_type", None),
        flag=getattr(opt, "flag", None),
        alias=getattr(opt, "alias", None),
        qa_whiteboard=getattr(opt, "qa_whiteboard", None),
        devel_whiteboard=getattr(opt, "devel_whiteboard", None),
        boolean_query=getattr(opt, "boolean_query", None),
        bug_severity=getattr(opt, "bug_severity", None),
        priority=getattr(opt, "priority", None),
        target_milestone=getattr(opt, "target_milestone", None),
        emailtype=opt.emailtype,
        booleantype=opt.booleantype,
        include_fields=include_fields,
        quicksearch=getattr(opt, "quicksearch", None),
        savedsearch=getattr(opt, "savedsearch", None),
        savedsearch_sharer_id=getattr(opt, "savedsearch_sharer_id", None),
        tags=getattr(opt, "tags", None))

    _merge_field_opts(built_query, opt, parser)

    built_query.update(q)
    q = built_query

    if not q:
        parser.error("'query' command requires additional arguments")
    if opt.test_return_result:
        return q
    return bz.query(q)


def _do_info(bz, opt):
    """
    Handle the 'info' subcommand
    """
    # All these commands call getproducts internally, so do it up front
    # with minimal include_fields for speed
    include_fields = ["name", "id"]
    if opt.versions:
        include_fields.append("versions")
    products = bz.getproducts(include_fields=include_fields)

    if opt.products:
        for name in sorted([p["name"] for p in products]):
            print(name)

    if opt.components:
        for name in sorted(bz.getcomponents(opt.components)):
            print(name)

    if opt.component_owners:
        # Looking up this info for rhbz 'Fedora' product is sloooow
        # since there are so many components. So delay getting this
        # info until as late as possible
        bz.refresh_products(names=[opt.component_owners],
                            include_fields=include_fields + [
                                "components.default_assigned_to",
                                "components.default_qa_contact",
                                "components.name",
                                "components.description"])

        component_details = bz.getcomponentsdetails(opt.component_owners)
        for c in sorted(component_details):
            print(to_encoding(u"%s: %s" %
                              (c, component_details[c]['initialowner'])))

    if opt.versions:
        for p in products:
            if p['name'] != opt.versions:
                continue
            if "versions" in p:
                for v in p['versions']:
                    print(to_encoding(v["name"]))
            break


def _convert_to_outputformat(output):
    fmt = ""

    if output == "normal":
        fmt = "%{__unicode__}"

    elif output == "ids":
        fmt = "%{id}"

    elif output == 'full':
        fmt += "%{__unicode__}\n"
        fmt += "Component: %{component}\n"
        fmt += "CC: %{cc}\n"
        fmt += "Blocked: %{blocks}\n"
        fmt += "Depends: %{depends_on}\n"
        fmt += "%{comments}\n"

    elif output == 'extra':
        fmt += "%{__unicode__}\n"
        fmt += " +Keywords: %{keywords}\n"
        fmt += " +QA Whiteboard: %{qa_whiteboard}\n"
        fmt += " +Status Whiteboard: %{status_whiteboard}\n"
        fmt += " +Devel Whiteboard: %{devel_whiteboard}\n"

    elif output == 'oneline':
        fmt += "#%{bug_id} %{status} %{assigned_to} %{component}\t"
        fmt += "[%{target_milestone}] %{flags} %{cve}"

    else:
        raise RuntimeError("Unknown output type '%s'" % output)

    return fmt


def _format_output(bz, opt, buglist):
    if opt.output == 'raw':
        buglist = bz.getbugs([b.bug_id for b in buglist])
        for b in buglist:
            print("Bugzilla %s: " % b.bug_id)
            for attrname in sorted(b.__dict__):
                print(to_encoding(u"ATTRIBUTE[%s]: %s" %
                                  (attrname, b.__dict__[attrname])))
            print("\n\n")
        return

    def bug_field(matchobj):
        # whiteboard and flag allow doing
        #   %{whiteboard:devel} and %{flag:needinfo}
        # That's what 'rest' matches
        (fieldname, rest) = matchobj.groups()

        if fieldname == "whiteboard" and rest:
            fieldname = rest + "_" + fieldname

        if fieldname == "flag" and rest:
            val = b.get_flag_status(rest)

        elif fieldname == "flags" or fieldname == "flags_requestee":
            tmpstr = []
            for f in getattr(b, "flags", []):
                requestee = f.get('requestee', "")
                if fieldname == "flags":
                    requestee = ""
                tmpstr.append("%s%s%s" % (f['name'], f['status'], requestee))
            val = ",".join(tmpstr)

        elif fieldname == "cve":
            cves = []
            for key in getattr(b, "keywords", []):
                # grab CVE from keywords and blockers
                if key.find("Security") == -1:
                    continue
                for bl in b.blocks:
                    cvebug = bz.getbug(bl)
                    for cb in cvebug.alias:
                        if cb.find("CVE") == -1:
                            continue
                        if cb.strip() not in cves:
                            cves.append(cb)
            val = ",".join(cves)

        elif fieldname == "comments":
            val = ""
            for c in getattr(b, "comments", []):
                val += ("\n* %s - %s:\n%s\n" %
                        (c['time'], c['author'], c['text']))

        elif fieldname == "__unicode__":
            val = b.__unicode__()
        else:
            val = getattr(b, fieldname, "")

        vallist = isinstance(val, list) and val or [val]
        val = ','.join([to_encoding(v) for v in vallist])

        return val

    for b in buglist:
        print(format_field_re.sub(bug_field, opt.outputformat))


def _parse_triset(vallist, checkplus=True, checkminus=True, checkequal=True,
                  splitcomma=False):
    add_val = []
    rm_val = []
    set_val = None

    def make_list(v):
        if not v:
            return []
        if splitcomma:
            return v.split(",")
        return [v]

    for val in isinstance(vallist, list) and vallist or [vallist]:
        val = val or ""

        if val.startswith("+") and checkplus:
            add_val += make_list(val[1:])
        elif val.startswith("-") and checkminus:
            rm_val += make_list(val[1:])
        elif val.startswith("=") and checkequal:
            # Intentionally overwrite this
            set_val = make_list(val[1:])
        else:
            add_val += make_list(val)

    return add_val, rm_val, set_val


def _do_new(bz, opt, parser):
    # Parse options that accept comma separated list
    def parse_multi(val):
        return _parse_triset(val, checkplus=False, checkminus=False,
                             checkequal=False, splitcomma=True)[0]

    ret = bz.build_createbug(
        blocks=parse_multi(opt.blocked) or None,
        cc=parse_multi(opt.cc) or None,
        component=opt.component or None,
        depends_on=parse_multi(opt.dependson) or None,
        description=opt.description or None,
        groups=parse_multi(opt.groups) or None,
        keywords=parse_multi(opt.keywords) or None,
        op_sys=opt.os or None,
        platform=opt.arch or None,
        priority=opt.priority or None,
        product=opt.product or None,
        severity=opt.severity or None,
        summary=opt.summary or None,
        url=opt.url or None,
        version=opt.version or None,
        assigned_to=opt.assigned_to or None,
        qa_contact=opt.qa_contact or None,
        sub_component=opt.sub_component or None,
        alias=opt.alias or None,
    )

    _merge_field_opts(ret, opt, parser)

    if opt.test_return_result:
        return ret

    b = bz.createbug(ret)
    b.refresh()
    return [b]


def _do_modify(bz, parser, opt, args):
    bugid_list = [bugid for a in args for bugid in a.split(',')]

    add_wb, rm_wb, set_wb = _parse_triset(opt.whiteboard)
    add_devwb, rm_devwb, set_devwb = _parse_triset(opt.devel_whiteboard)
    add_intwb, rm_intwb, set_intwb = _parse_triset(opt.internal_whiteboard)
    add_qawb, rm_qawb, set_qawb = _parse_triset(opt.qa_whiteboard)

    add_blk, rm_blk, set_blk = _parse_triset(opt.blocked, splitcomma=True)
    add_deps, rm_deps, set_deps = _parse_triset(opt.dependson, splitcomma=True)
    add_key, rm_key, set_key = _parse_triset(opt.keywords)
    add_cc, rm_cc, ignore = _parse_triset(opt.cc,
                                          checkplus=False,
                                          checkequal=False)
    add_groups, rm_groups, ignore = _parse_triset(opt.groups,
                                                  checkequal=False,
                                                  splitcomma=True)
    add_tags, rm_tags, ignore = _parse_triset(opt.tags, checkequal=False)

    status = opt.status or None
    if opt.dupeid is not None:
        opt.close = "DUPLICATE"
    if opt.close:
        status = "CLOSED"

    update = bz.build_update(
        assigned_to=opt.assignee or None,
        comment=opt.comment or None,
        comment_private=opt.private or None,
        component=opt.component or None,
        product=opt.product or None,
        blocks_add=add_blk or None,
        blocks_remove=rm_blk or None,
        blocks_set=set_blk,
        url=opt.url or None,
        cc_add=add_cc or None,
        cc_remove=rm_cc or None,
        depends_on_add=add_deps or None,
        depends_on_remove=rm_deps or None,
        depends_on_set=set_deps,
        groups_add=add_groups or None,
        groups_remove=rm_groups or None,
        keywords_add=add_key or None,
        keywords_remove=rm_key or None,
        keywords_set=set_key,
        op_sys=opt.os or None,
        platform=opt.arch or None,
        priority=opt.priority or None,
        qa_contact=opt.qa_contact or None,
        severity=opt.severity or None,
        status=status,
        summary=opt.summary or None,
        version=opt.version or None,
        reset_assigned_to=opt.reset_assignee,
        reset_qa_contact=opt.reset_qa_contact,
        resolution=opt.close or None,
        target_release=opt.target_release or None,
        target_milestone=opt.target_milestone or None,
        dupe_of=opt.dupeid or None,
        fixed_in=opt.fixed_in or None,
        whiteboard=set_wb and set_wb[0] or None,
        devel_whiteboard=set_devwb and set_devwb[0] or None,
        internal_whiteboard=set_intwb and set_intwb[0] or None,
        qa_whiteboard=set_qawb and set_qawb[0] or None,
        sub_component=opt.sub_component or None,
        alias=opt.alias or None,
    )

    flags = []
    if opt.flag:
        # Convert "foo+" to tuple ("foo", "+")
        for f in opt.flag:
            flags.append({"name": f[:-1], "status": f[-1]})

    # We make this a little convoluted to facilitate unit testing
    wbmap = {
        "status": (add_wb, rm_wb),
        "internal": (add_intwb, rm_intwb),
        "qa": (add_qawb, rm_qawb),
        "devel": (add_devwb, rm_devwb),
    }

    for k, v in wbmap.copy().items():
        if not v[0] and not v[1]:
            del(wbmap[k])

    _merge_field_opts(update, opt, parser)

    log.debug("update bug dict=%s", update)
    log.debug("update flags dict=%s", flags)
    log.debug("update whiteboard dict=%s", wbmap)

    if not any([flags, update, wbmap, add_tags, rm_tags]):
        parser.error("'modify' command requires additional arguments")

    if opt.test_return_result:
        return (update, flags, wbmap, add_tags, rm_tags)

    if flags:
        ret = bz.update_flags(bugid_list, flags)
        log.debug("bz.update_flags returned=%s", ret)
    if add_tags or rm_tags:
        ret = bz.update_tags(bugid_list,
            tags_add=add_tags, tags_remove=rm_tags)
        log.debug("bz.update_tags returned=%s", ret)
    if update:
        ret = bz.update_bugs(bugid_list, update)
        log.debug("bz.update_bugs returned=%s", ret)

    if not wbmap:
        return

    # Now for the things we can't blindly batch.
    # This could be improved to batch identical updates.
    log.debug("Adjusting whiteboard fields one by one")
    for bug in bz.getbugs(bugid_list):
        for wbtype, (add_list, rm_list) in wbmap.items():
            for tag in add_list:
                bug.addtag(tag, which=wbtype)
            for tag in rm_list:
                bug.deltag(tag, which=wbtype)


def _do_get_attach(bz, opt, parser, args):
    if args:
        parser.error("Extra args '%s' not used for getting attachments" %
                     args)

    for bug in bz.getbugs(opt.getall):
        opt.get += bug.get_attachment_ids()

    for attid in set(opt.get):
        att = bz.openattachment(attid)
        outfile = open_without_clobber(att.name, "wb")
        data = att.read(4096)
        while data:
            outfile.write(data)
            data = att.read(4096)
        print("Wrote %s" % outfile.name)

    return


def _do_set_attach(bz, opt, parser, args):
    if not args:
        parser.error("Bug ID must be specified for setting attachments")

    if sys.stdin.isatty():
        if not opt.file:
            parser.error("--file must be specified")
        fileobj = open(opt.file)
    else:
        # piped input on stdin
        if not opt.desc:
            parser.error("--description must be specified if passing "
                         "file on stdin")

        fileobj = tempfile.NamedTemporaryFile(prefix="bugzilla-attach.")
        data = sys.stdin.read(4096)

        while data:
            fileobj.write(data.encode(locale.getpreferredencoding()))
            data = sys.stdin.read(4096)
        fileobj.seek(0)

    kwargs = {}
    if opt.file:
        kwargs["filename"] = os.path.basename(opt.file)
    if opt.type:
        kwargs["contenttype"] = opt.type
    if opt.type in ["text/x-patch"]:
        kwargs["ispatch"] = True
    desc = opt.desc or os.path.basename(fileobj.name)

    # Upload attachments
    for bugid in args:
        attid = bz.attachfile(bugid, fileobj, desc, **kwargs)
        print("Created attachment %i on bug %s" % (attid, bugid))


#################
# Main function #
#################

def main(bzinstance=None):
    parser = setup_parser()
    (global_opt, args) = parser.parse_args()

    if global_opt.generate_man:
        generate_man_page()
        return 0

    if global_opt.debug:
        log.setLevel(DEBUG)
    elif global_opt.verbose:
        log.setLevel(INFO)
    else:
        log.setLevel(WARN)

    log.debug("Launched with command line: %s", " ".join(sys.argv))

    # Get our action
    if len(args) == 0:
        parser.error("No command specified, command must be one of: %s" %
                     ', '.join(cmdlist))
    action = args.pop(0)
    if action not in cmdlist:
        parser.error("Unknown command '%s', command must be one of: %s" %
                     (action, ', '.join(cmdlist)))

    # Parse action-specific args
    action_parser = setup_action_parser(action)
    (opt, args) = action_parser.parse_args(args)

    # Connect to bugzilla
    log.info('Connecting to %s', global_opt.bugzilla)

    if global_opt.bztype == 'auto':
        log.info('Autodetecting Bugzilla type')
        bzclass = bugzilla.Bugzilla
    elif global_opt.bztype in bugzilla.classlist:
        log.info('Using Bugzilla class %s', global_opt.bztype)
        bzclass = getattr(bugzilla, global_opt.bztype)
    else:
        parser.error("bztype must be one of: %s" % str(bugzilla.classlist))

    if bzinstance:
        bz = bzinstance
    else:
        if global_opt.cache_credentials:
            cookiefile = global_opt.cookiefile or -1
            tokenfile = global_opt.tokenfile or -1
        else:
            cookiefile = None
            tokenfile = None
        bz = bzclass(url=global_opt.bugzilla,
                     cookiefile=cookiefile,
                     tokenfile=tokenfile,
                     sslverify=global_opt.sslverify)


    # Handle 'login' action
    is_login_command = (action == 'login')
    force_login = is_login_command or global_opt.login

    if force_login:
        if is_login_command:
            if len(args) == 2:
                (global_opt.user, global_opt.password) = args
            elif len(args) == 1:
                (global_opt.user, ) = args
            elif len(args) > 2:
                parser.error("Too many arguments for login")

    try:
        if force_login:
            bz.interactive_login(
                global_opt.user, global_opt.password, force_login)
        if is_login_command:
            print("Login successful.")
            sys.exit(0)
    except bugzilla.BugzillaError:
        print(str(sys.exc_info()[1]))
        sys.exit(1)

    if global_opt.ensure_logged_in and not bz.logged_in:
        print("--ensure-logged-in passed but you aren't logged in to %s" %
            bz.url)
        sys.exit(1)


    ###########################
    # Run the actual commands #
    ###########################

    if hasattr(opt, "outputformat"):
        if not opt.outputformat and opt.output not in ['raw', None]:
            opt.outputformat = _convert_to_outputformat(opt.output)

    buglist = []
    if action == 'info':
        if args:
            parser.error("Extra arguments '%s'" % args)

        if not (opt.products or
                opt.components or
                opt.component_owners or
                opt.versions):
            parser.error("'info' command requires additional arguments")

        _do_info(bz, opt)

    elif action == 'query':
        if args:
            parser.error("Extra arguments '%s'" % args)

        buglist = _do_query(bz, opt, parser)
        if opt.test_return_result:
            return buglist

    elif action == 'new':
        if args:
            parser.error("Extra arguments '%s'" % args)
        buglist = _do_new(bz, opt, parser)
        if opt.test_return_result:
            return buglist

    elif action == 'attach':
        if opt.get or opt.getall:
            _do_get_attach(bz, opt, parser, args)
        else:
            _do_set_attach(bz, opt, parser, args)

    elif action == 'modify':
        if not args:
            parser.error('No bug IDs given '
                         '(maybe you forgot an argument somewhere?)')

        modout = _do_modify(bz, parser, opt, args)
        if opt.test_return_result:
            return modout
    else:
        raise RuntimeError("Unexpected action '%s'" % action)

    # If we're doing new/query/modify, output our results
    if action in ['new', 'query']:
        _format_output(bz, opt, buglist)


if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        log.debug("", exc_info=True)
        print("\nExited at user request.")
        sys.exit(1)
    except (Fault, bugzilla.BugzillaError):
        e = sys.exc_info()[1]
        log.debug("", exc_info=True)
        print("\nServer error: %s" % str(e))
        sys.exit(3)
    except ProtocolError:
        e = sys.exc_info()[1]
        log.debug("", exc_info=True)
        print("\nInvalid server response: %d %s" % (e.errcode, e.errmsg))
        # Detect redirect
        redir = (e.headers and 'location' in e.headers)
        if redir:
            print("\nServer was attempting a redirect. Try: "
                  "  bugzilla --bugzilla %s ..." % redir)
        sys.exit(4)
    except requests.exceptions.SSLError:
        e = sys.exc_info()[1]
        log.debug("", exc_info=True)

        # Give SSL recommendations
        print("SSL error: %s" % e)
        print("\nIf you trust the remote server, you can work "
              "around this error with:\n"
              "  bugzilla --nosslverify ...")
        sys.exit(4)
    except (socket.error,
            requests.exceptions.HTTPError,
            requests.exceptions.ConnectionError):
        e = sys.exc_info()[1]
        log.debug("", exc_info=True)
        print("\nConnection lost/failed: %s" % str(e))
        sys.exit(2)