Back to index

apport  2.4
launchpad.py
Go to the documentation of this file.
00001 # vim: set fileencoding=UTF-8 :
00002 '''Crash database implementation for Launchpad.'''
00003 
00004 # Copyright (C) 2007 - 2009 Canonical Ltd.
00005 # Authors: Martin Pitt <martin.pitt@ubuntu.com> and Markus Korn <thekorn@gmx.de>
00006 #
00007 # This program is free software; you can redistribute it and/or modify it
00008 # under the terms of the GNU General Public License as published by the
00009 # Free Software Foundation; either version 2 of the License, or (at your
00010 # option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
00011 # the full text of the license.
00012 
00013 import tempfile, os.path, re, gzip, sys, email, time
00014 
00015 from io import BytesIO
00016 
00017 if sys.version_info.major == 2:
00018     from urllib2 import HTTPSHandler, Request, build_opener
00019     from httplib import HTTPSConnection
00020     from urllib import urlencode, urlopen
00021     (HTTPSHandler, Request, build_opener, HTTPSConnection, urlencode, urlopen)  # pyflakes
00022 else:
00023     from urllib.request import HTTPSHandler, Request, build_opener, urlopen
00024     from urllib.parse import urlencode
00025     from http.client import HTTPSConnection
00026 
00027 try:
00028     from launchpadlib.errors import HTTPError
00029     from launchpadlib.launchpad import Launchpad
00030     Launchpad  # pyflakes
00031 except ImportError:
00032     # if launchpadlib is not available, only client-side reporting will work
00033     Launchpad = None
00034 
00035 import apport.crashdb
00036 import apport
00037 
00038 default_credentials_path = os.path.expanduser('~/.cache/apport/launchpad.credentials')
00039 
00040 
00041 def filter_filename(attachments):
00042     for attachment in attachments:
00043         try:
00044             f = attachment.data.open()
00045         except HTTPError:
00046             apport.error('Broken attachment on bug, ignoring')
00047             continue
00048         name = f.filename
00049         if name.endswith('.txt') or name.endswith('.gz'):
00050             yield f
00051 
00052 
00053 def id_set(tasks):
00054     # same as set(int(i.bug.id) for i in tasks) but faster
00055     return set(int(i.self_link.split('/').pop()) for i in tasks)
00056 
00057 
00058 class CrashDatabase(apport.crashdb.CrashDatabase):
00059     '''Launchpad implementation of crash database interface.'''
00060 
00061     def __init__(self, auth, options):
00062         '''Initialize Launchpad crash database.
00063 
00064         You need to specify a launchpadlib-style credentials file to
00065         access launchpad. If you supply None, it will use
00066         default_credentials_path (~/.cache/apport/launchpad.credentials).
00067 
00068         Recognized options are:
00069         - distro: Name of the distribution in Launchpad
00070         - project: Name of the project in Launchpad
00071         (Note that exactly one of "distro" or "project" must be given.)
00072         - launchpad_instance: If set, this uses the given launchpad instance
00073           instead of production (optional). This can be overriden or set by
00074           $APPORT_LAUNCHPAD_INSTANCE environment.
00075         - cache_dir: Path to a permanent cache directory; by default it uses a
00076           temporary one. (optional). This can be overridden or set by
00077           $APPORT_LAUNCHPAD_CACHE environment.
00078         - escalation_subscription: This subscribes the given person or team to
00079           a bug once it gets the 10th duplicate.
00080         - escalation_tag: This adds the given tag to a bug once it gets more
00081           than 10 duplicates.
00082         - initial_subscriber: The Launchpad user which gets subscribed to newly
00083           filed bugs (default: "apport"). It should be a bot user which the
00084           crash-digger instance runs as, as this will get to see all bug
00085           details immediately.
00086         - triaging_team: The Launchpad user/team which gets subscribed after
00087           updating a crash report bug by the retracer (default:
00088           "ubuntu-crashes-universe")
00089         '''
00090         if os.getenv('APPORT_LAUNCHPAD_INSTANCE'):
00091             options['launchpad_instance'] = os.getenv(
00092                 'APPORT_LAUNCHPAD_INSTANCE')
00093         if not auth:
00094             lp_instance = options.get('launchpad_instance')
00095             if lp_instance:
00096                 auth = default_credentials_path + '.' + lp_instance.split('://', 1)[-1]
00097             else:
00098                 auth = default_credentials_path
00099         apport.crashdb.CrashDatabase.__init__(self, auth, options)
00100 
00101         self.distro = options.get('distro')
00102         if self.distro:
00103             assert 'project' not in options, 'Must not set both "project" and "distro" option'
00104         else:
00105             assert 'project' in options, 'Need to have either "project" or "distro" option'
00106 
00107         self.arch_tag = 'need-%s-retrace' % apport.packaging.get_system_architecture()
00108         self.options = options
00109         self.auth = auth
00110         assert self.auth
00111 
00112         self.__launchpad = None
00113         self.__lp_distro = None
00114         self.__lpcache = os.getenv('APPORT_LAUNCHPAD_CACHE', options.get('cache_dir'))
00115 
00116     @property
00117     def launchpad(self):
00118         '''Return Launchpad instance.'''
00119 
00120         if self.__launchpad:
00121             return self.__launchpad
00122 
00123         if Launchpad is None:
00124             sys.stderr.write('ERROR: The launchpadlib Python module is not installed. This functionality is not available.\n')
00125             sys.exit(1)
00126 
00127         if self.options.get('launchpad_instance'):
00128             launchpad_instance = self.options.get('launchpad_instance')
00129         else:
00130             launchpad_instance = 'production'
00131 
00132         auth_dir = os.path.dirname(self.auth)
00133         if auth_dir and not os.path.isdir(auth_dir):
00134             os.makedirs(auth_dir)
00135 
00136         try:
00137             self.__launchpad = Launchpad.login_with('apport-collect',
00138                                                     launchpad_instance,
00139                                                     launchpadlib_dir=self.__lpcache,
00140                                                     allow_access_levels=['WRITE_PRIVATE'],
00141                                                     credentials_file=self.auth,
00142                                                     version='1.0')
00143         except Exception as e:
00144             if hasattr(e, 'content'):
00145                 msg = e.content
00146             else:
00147                 msg = str(e)
00148             apport.error('connecting to Launchpad failed: %s\nYou can reset the credentials by removing the file "%s"', msg, self.auth)
00149             sys.exit(99)  # transient error
00150 
00151         return self.__launchpad
00152 
00153     def _get_distro_tasks(self, tasks):
00154         if not self.distro:
00155             raise StopIteration
00156 
00157         for t in tasks:
00158             if t.bug_target_name.lower() == self.distro or \
00159                     re.match('^.+\(%s.*\)$' % self.distro, t.bug_target_name.lower()):
00160                 yield t
00161 
00162     @property
00163     def lp_distro(self):
00164         if self.__lp_distro is None:
00165             if self.distro:
00166                 self.__lp_distro = self.launchpad.distributions[self.distro]
00167             elif 'project' in self.options:
00168                 self.__lp_distro = self.launchpad.projects[self.options['project']]
00169             else:
00170                 raise SystemError('distro or project needs to be specified in crashdb options')
00171 
00172         return self.__lp_distro
00173 
00174     def upload(self, report, progress_callback=None):
00175         '''Upload given problem report return a handle for it.
00176 
00177         This should happen noninteractively.
00178 
00179         If the implementation supports it, and a function progress_callback is
00180         passed, that is called repeatedly with two arguments: the number of
00181         bytes already sent, and the total number of bytes to send. This can be
00182         used to provide a proper upload progress indication on frontends.
00183         '''
00184         assert self.accepts(report)
00185 
00186         blob_file = self._generate_upload_blob(report)
00187         ticket = upload_blob(blob_file, progress_callback, hostname=self.get_hostname())
00188         blob_file.close()
00189         assert ticket
00190         return ticket
00191 
00192     def get_hostname(self):
00193         '''Return the hostname for the Launchpad instance.'''
00194 
00195         launchpad_instance = self.options.get('launchpad_instance')
00196         if launchpad_instance:
00197             if launchpad_instance == 'staging':
00198                 hostname = 'staging.launchpad.net'
00199             else:
00200                 hostname = 'launchpad.dev'
00201         else:
00202             hostname = 'launchpad.net'
00203         return hostname
00204 
00205     def get_comment_url(self, report, handle):
00206         '''Return an URL that should be opened after report has been uploaded
00207         and upload() returned handle.
00208 
00209         Should return None if no URL should be opened (anonymous filing without
00210         user comments); in that case this function should do whichever
00211         interactive steps it wants to perform.'''
00212 
00213         args = {}
00214         title = report.get('Title', report.standard_title())
00215         if title:
00216             # always use UTF-8 encoding, urlencode() blows up otherwise in
00217             # python 2.7
00218             if type(title) != type(b''):
00219                 title = title.encode('UTF-8')
00220             args['field.title'] = title
00221 
00222         hostname = self.get_hostname()
00223 
00224         project = self.options.get('project')
00225 
00226         if not project:
00227             if 'SourcePackage' in report:
00228                 return 'https://bugs.%s/%s/+source/%s/+filebug/%s?%s' % (
00229                     hostname, self.distro, report['SourcePackage'], handle, urlencode(args))
00230             else:
00231                 return 'https://bugs.%s/%s/+filebug/%s?%s' % (
00232                     hostname, self.distro, handle, urlencode(args))
00233         else:
00234             return 'https://bugs.%s/%s/+filebug/%s?%s' % (
00235                 hostname, project, handle, urlencode(args))
00236 
00237     def get_id_url(self, report, id):
00238         '''Return URL for a given report ID.
00239 
00240         The report is passed in case building the URL needs additional
00241         information from it, such as the SourcePackage name.
00242 
00243         Return None if URL is not available or cannot be determined.
00244         '''
00245         return 'https://bugs.launchpad.net/bugs/' + str(id)
00246 
00247     def download(self, id):
00248         '''Download the problem report from given ID and return a Report.'''
00249 
00250         report = apport.Report()
00251         b = self.launchpad.bugs[id]
00252 
00253         # parse out fields from summary
00254         m = re.search(r'(ProblemType:.*)$', b.description, re.S)
00255         if not m:
00256             m = re.search(r'^--- \r?$[\r\n]*(.*)', b.description, re.M | re.S)
00257         assert m, 'bug description must contain standard apport format data'
00258 
00259         description = m.group(1).encode('UTF-8').replace('\xc2\xa0', ' ').replace('\r\n', '\n')
00260 
00261         if '\n\n' in description:
00262             # this often happens, remove all empty lines between top and
00263             # 'Uname'
00264             if 'Uname:' in description:
00265                 # this will take care of bugs like LP #315728 where stuff
00266                 # is added after the apport data
00267                 (part1, part2) = description.split('Uname:', 1)
00268                 description = part1.replace('\n\n', '\n') + 'Uname:' \
00269                     + part2.split('\n\n', 1)[0]
00270             else:
00271                 # just parse out the Apport block; e. g. LP #269539
00272                 description = description.split('\n\n', 1)[0]
00273 
00274         report.load(BytesIO(description))
00275 
00276         if 'Date' not in report:
00277             # We had not submitted this field for a while, claiming it
00278             # redundant. But it is indeed required for up-to-the-minute
00279             # comparison with log files, etc. For backwards compatibility with
00280             # those reported bugs, read the creation date
00281             try:
00282                 report['Date'] = b.date_created.ctime()
00283             except AttributeError:
00284                 # support older wadllib API which returned strings
00285                 report['Date'] = b.date_created
00286         if 'ProblemType' not in report:
00287             if 'apport-bug' in b.tags:
00288                 report['ProblemType'] = 'Bug'
00289             elif 'apport-crash' in b.tags:
00290                 report['ProblemType'] = 'Crash'
00291             elif 'apport-kernelcrash' in b.tags:
00292                 report['ProblemType'] = 'KernelCrash'
00293             elif 'apport-package' in b.tags:
00294                 report['ProblemType'] = 'Package'
00295             else:
00296                 raise ValueError('cannot determine ProblemType from tags: ' + str(b.tags))
00297 
00298         report['Tags'] = ' '.join(b.tags)
00299 
00300         if 'Title' in report:
00301             report['OriginalTitle'] = report['Title']
00302 
00303         report['Title'] = b.title
00304 
00305         for attachment in filter_filename(b.attachments):
00306             key, ext = os.path.splitext(attachment.filename)
00307             # ignore attachments with invalid keys
00308             try:
00309                 report[key] = ''
00310             except:
00311                 continue
00312             if ext == '.txt':
00313                 report[key] = attachment.read()
00314             elif ext == '.gz':
00315                 try:
00316                     report[key] = gzip.GzipFile(fileobj=attachment).read()
00317                 except IOError as e:
00318                     # some attachments are only called .gz, but are
00319                     # uncompressed (LP #574360)
00320                     if 'Not a gzip' not in str(e):
00321                         raise
00322                     attachment.seek(0)
00323                     report[key] = attachment.read()
00324             else:
00325                 raise Exception('Unknown attachment type: ' + attachment.filename)
00326         return report
00327 
00328     def update(self, id, report, comment, change_description=False,
00329                attachment_comment=None, key_filter=None):
00330         '''Update the given report ID with all data from report.
00331 
00332         This creates a text comment with the "short" data (see
00333         ProblemReport.write_mime()), and creates attachments for all the
00334         bulk/binary data.
00335 
00336         If change_description is True, and the crash db implementation supports
00337         it, the short data will be put into the description instead (like in a
00338         new bug).
00339 
00340         comment will be added to the "short" data. If attachment_comment is
00341         given, it will be added to the attachment uploads.
00342 
00343         If key_filter is a list or set, then only those keys will be added.
00344         '''
00345         bug = self.launchpad.bugs[id]
00346 
00347         if key_filter:
00348             skip_keys = set(report.keys()) - set(key_filter)
00349         else:
00350             skip_keys = None
00351 
00352         # we want to reuse the knowledge of write_mime() with all its different input
00353         # types and output formatting; however, we have to dissect the mime ourselves,
00354         # since we can't just upload it as a blob
00355         mime = tempfile.TemporaryFile()
00356         report.write_mime(mime, skip_keys=skip_keys)
00357         mime.flush()
00358         mime.seek(0)
00359         msg = email.message_from_file(mime)
00360         msg_iter = msg.walk()
00361 
00362         # first part is the multipart container
00363         part = msg_iter.next()
00364         assert part.is_multipart()
00365 
00366         # second part should be an inline text/plain attachments with all short
00367         # fields
00368         part = msg_iter.next()
00369         assert not part.is_multipart()
00370         assert part.get_content_type() == 'text/plain'
00371 
00372         if not key_filter:
00373             # when we update a complete report, we are updating an existing bug
00374             # with apport-collect
00375             x = bug.tags[:]  # LP#254901 workaround
00376             x.append('apport-collected')
00377             # add any tags (like the release) to the bug
00378             if 'Tags' in report:
00379                 x += report['Tags'].lower().split()
00380             bug.tags = x
00381             bug.lp_save()
00382             bug = self.launchpad.bugs[id]  # fresh bug object, LP#336866 workaround
00383 
00384         # short text data
00385         if change_description:
00386             bug.description = bug.description + '\n--- \n' + part.get_payload(decode=True).decode('UTF-8', 'replace')
00387             bug.lp_save()
00388         else:
00389             bug.newMessage(content=part.get_payload(decode=True), subject=comment)
00390 
00391         # other parts are the attachments:
00392         for part in msg_iter:
00393             # print '   attachment: %s...' % part.get_filename()
00394             bug.addAttachment(comment=attachment_comment or '',
00395                               description=part.get_filename(),
00396                               content_type=None,
00397                               data=part.get_payload(decode=True),
00398                               filename=part.get_filename(), is_patch=False)
00399 
00400     def update_traces(self, id, report, comment=''):
00401         '''Update the given report ID for retracing results.
00402 
00403         This updates Stacktrace, ThreadStacktrace, StacktraceTop,
00404         and StacktraceSource. You can also supply an additional comment.
00405         '''
00406         apport.crashdb.CrashDatabase.update_traces(self, id, report, comment)
00407 
00408         bug = self.launchpad.bugs[id]
00409         # ensure it's assigned to a package
00410         if 'SourcePackage' in report:
00411             for task in bug.bug_tasks:
00412                 if task.target.resource_type_link.endswith('#distribution'):
00413                     task.target = self.lp_distro.getSourcePackage(name=report['SourcePackage'])
00414                     task.lp_save()
00415                     bug = self.launchpad.bugs[id]
00416                     break
00417 
00418         # remove core dump if stack trace is usable
00419         if report.has_useful_stacktrace():
00420             for a in bug.attachments:
00421                 if a.title == 'CoreDump.gz':
00422                     try:
00423                         a.removeFromBug()
00424                     except HTTPError:
00425                         pass  # LP#249950 workaround
00426             try:
00427                 task = self._get_distro_tasks(bug.bug_tasks).next()
00428                 if task.importance == 'Undecided':
00429                     task.importance = 'Medium'
00430                     task.lp_save()
00431             except StopIteration:
00432                 pass  # no distro tasks
00433 
00434             # update bug title with retraced function name
00435             fn = report.stacktrace_top_function()
00436             if fn:
00437                 m = re.match('^(.*crashed with SIG.* in )([^( ]+)(\(\).*$)', bug.title)
00438                 if m and m.group(2) != fn:
00439                     bug.title = m.group(1) + fn + m.group(3)
00440                     try:
00441                         bug.lp_save()
00442                     except HTTPError:
00443                         pass  # LP#336866 workaround
00444                     bug = self.launchpad.bugs[id]
00445 
00446         self._subscribe_triaging_team(bug, report)
00447 
00448     def get_distro_release(self, id):
00449         '''Get 'DistroRelease: <release>' from the given report ID and return
00450         it.'''
00451         bug = self.launchpad.bugs[id]
00452         m = re.search('DistroRelease: ([-a-zA-Z0-9.+/ ]+)', bug.description)
00453         if m:
00454             return m.group(1)
00455         raise ValueError('URL does not contain DistroRelease: field')
00456 
00457     def get_affected_packages(self, id):
00458         '''Return list of affected source packages for given ID.'''
00459 
00460         bug_target_re = re.compile(
00461             r'/%s/(?:(?P<suite>[^/]+)/)?\+source/(?P<source>[^/]+)$' % self.distro)
00462 
00463         bug = self.launchpad.bugs[id]
00464         result = []
00465 
00466         for task in bug.bug_tasks:
00467             match = bug_target_re.search(task.target.self_link)
00468             if not match:
00469                 continue
00470             if task.status in ('Invalid', "Won't Fix", 'Fix Released'):
00471                 continue
00472             result.append(match.group('source'))
00473         return result
00474 
00475     def is_reporter(self, id):
00476         '''Check whether the user is the reporter of given ID.'''
00477 
00478         bug = self.launchpad.bugs[id]
00479         return bug.owner.name == self.launchpad.me.name
00480 
00481     def can_update(self, id):
00482         '''Check whether the user is eligible to update a report.
00483 
00484         A user should add additional information to an existing ID if (s)he is
00485         the reporter or subscribed, the bug is open, not a duplicate, etc. The
00486         exact policy and checks should be done according to  the particular
00487         implementation.
00488         '''
00489         bug = self.launchpad.bugs[id]
00490         if bug.duplicate_of:
00491             return False
00492 
00493         if bug.owner.name == self.launchpad.me.name:
00494             return True
00495 
00496         # check subscription
00497         me = self.launchpad.me.self_link
00498         for sub in bug.subscriptions.entries:
00499             if sub['person_link'] == me:
00500                 return True
00501 
00502         return False
00503 
00504     def get_unretraced(self):
00505         '''Return an ID set of all crashes which have not been retraced yet and
00506         which happened on the current host architecture.'''
00507         try:
00508             bugs = self.lp_distro.searchTasks(tags=self.arch_tag, created_since='2011-08-01')
00509             return id_set(bugs)
00510         except Exception as e:
00511             apport.error('connecting to Launchpad failed: %s', str(e))
00512             sys.exit(99)  # transient error
00513 
00514     def get_dup_unchecked(self):
00515         '''Return an ID set of all crashes which have not been checked for
00516         being a duplicate.
00517 
00518         This is mainly useful for crashes of scripting languages such as
00519         Python, since they do not need to be retraced. It should not return
00520         bugs that are covered by get_unretraced().'''
00521 
00522         try:
00523             bugs = self.lp_distro.searchTasks(tags='need-duplicate-check', created_since='2011-08-01')
00524             return id_set(bugs)
00525         except Exception as e:
00526             apport.error('connecting to Launchpad failed: %s', str(e))
00527             sys.exit(99)  # transient error
00528 
00529     def get_unfixed(self):
00530         '''Return an ID set of all crashes which are not yet fixed.
00531 
00532         The list must not contain bugs which were rejected or duplicate.
00533 
00534         This function should make sure that the returned list is correct. If
00535         there are any errors with connecting to the crash database, it should
00536         raise an exception (preferably IOError).'''
00537 
00538         bugs = self.lp_distro.searchTasks(tags='apport-crash')
00539         return id_set(bugs)
00540 
00541     def _get_source_version(self, package):
00542         '''Return the version of given source package in the latest release of
00543         given distribution.
00544 
00545         If 'distro' is None, we will look for a launchpad project .
00546         '''
00547         sources = self.lp_distro.main_archive.getPublishedSources(
00548             exact_match=True,
00549             source_name=package,
00550             distro_series=self.lp_distro.current_series
00551         )
00552         # first element is the latest one
00553         return sources[0].source_package_version
00554 
00555     def get_fixed_version(self, id):
00556         '''Return the package version that fixes a given crash.
00557 
00558         Return None if the crash is not yet fixed, or an empty string if the
00559         crash is fixed, but it cannot be determined by which version. Return
00560         'invalid' if the crash report got invalidated, such as closed a
00561         duplicate or rejected.
00562 
00563         This function should make sure that the returned result is correct. If
00564         there are any errors with connecting to the crash database, it should
00565         raise an exception (preferably IOError).
00566         '''
00567         # do not do version tracking yet; for that, we need to get the current
00568         # distrorelease and the current package version in that distrorelease
00569         # (or, of course, proper version tracking in Launchpad itself)
00570 
00571         try:
00572             b = self.launchpad.bugs[id]
00573         except KeyError:
00574             return 'invalid'
00575 
00576         if b.duplicate_of:
00577             return 'invalid'
00578 
00579         tasks = list(b.bug_tasks)  # just fetch it once
00580 
00581         if self.distro:
00582             distro_identifier = '(%s)' % self.distro.lower()
00583             fixed_tasks = filter(lambda task: task.status == 'Fix Released' and
00584                                  distro_identifier in task.bug_target_display_name.lower(), tasks)
00585 
00586             if not fixed_tasks:
00587                 fixed_distro = filter(lambda task: task.status == 'Fix Released' and
00588                                       task.bug_target_name.lower() == self.distro.lower(), tasks)
00589                 if fixed_distro:
00590                     # fixed in distro inself (without source package)
00591                     return ''
00592 
00593             if len(fixed_tasks) > 1:
00594                 apport.warning('There is more than one task fixed in %s %s, using first one to determine fixed version', self.distro, id)
00595                 return ''
00596 
00597             if fixed_tasks:
00598                 task = fixed_tasks.pop()
00599                 try:
00600                     return self._get_source_version(task.bug_target_display_name.split()[0])
00601                 except IndexError:
00602                     # source does not exist any more
00603                     return 'invalid'
00604             else:
00605                 # check if there only invalid ones
00606                 invalid_tasks = filter(lambda task: task.status in ('Invalid', "Won't Fix", 'Expired') and
00607                                        distro_identifier in task.bug_target_display_name.lower(), tasks)
00608                 if invalid_tasks:
00609                     non_invalid_tasks = filter(
00610                         lambda task: task.status not in ('Invalid', "Won't Fix", 'Expired') and
00611                         distro_identifier in task.bug_target_display_name.lower(), tasks)
00612                     if not non_invalid_tasks:
00613                         return 'invalid'
00614         else:
00615             fixed_tasks = filter(lambda task: task.status == 'Fix Released', tasks)
00616             if fixed_tasks:
00617                 # TODO: look for current series
00618                 return ''
00619             # check if there any invalid ones
00620             if filter(lambda task: task.status == 'Invalid', tasks):
00621                 return 'invalid'
00622 
00623         return None
00624 
00625     def duplicate_of(self, id):
00626         '''Return master ID for a duplicate bug.
00627 
00628         If the bug is not a duplicate, return None.
00629         '''
00630         b = self.launchpad.bugs[id].duplicate_of
00631         if b:
00632             return b.id
00633         else:
00634             return None
00635 
00636     def close_duplicate(self, report, id, master_id):
00637         '''Mark a crash id as duplicate of given master ID.
00638 
00639         If master is None, id gets un-duplicated.
00640         '''
00641         bug = self.launchpad.bugs[id]
00642 
00643         if master_id:
00644             assert id != master_id, 'cannot mark bug %s as a duplicate of itself' % str(id)
00645 
00646             # check whether the master itself is a dup
00647             master = self.launchpad.bugs[master_id]
00648             if master.duplicate_of:
00649                 master = master.duplicate_of
00650                 master_id = master.id
00651                 if master.id == id:
00652                     # this happens if the bug was manually duped to a newer one
00653                     apport.warning('Bug %i was manually marked as a dupe of newer bug %i, not closing as duplicate',
00654                                    id, master_id)
00655                     return
00656 
00657             for a in bug.attachments:
00658                 if a.title in ('CoreDump.gz', 'Stacktrace.txt',
00659                                'ThreadStacktrace.txt', 'ProcMaps.txt',
00660                                'ProcStatus.txt', 'Registers.txt',
00661                                'Disassembly.txt'):
00662                     try:
00663                         a.removeFromBug()
00664                     except HTTPError:
00665                         pass  # LP#249950 workaround
00666 
00667             bug = self.launchpad.bugs[id]  # fresh bug object, LP#336866 workaround
00668             bug.newMessage(content='Thank you for taking the time to report this crash and helping \
00669 to make this software better.  This particular crash has already been reported and \
00670 is a duplicate of bug #%i, so is being marked as such.  Please look at the \
00671 other bug report to see if there is any missing information that you can \
00672 provide, or to see if there is a workaround for the bug.  Additionally, any \
00673 further discussion regarding the bug should occur in the other report.  \
00674 Please continue to report any other bugs you may find.' % master_id,
00675                            subject='This bug is a duplicate')
00676 
00677             bug = self.launchpad.bugs[id]  # refresh, LP#336866 workaround
00678             if bug.private:
00679                 bug.private = False
00680 
00681             # set duplicate last, since we cannot modify already dup'ed bugs
00682             if not bug.duplicate_of:
00683                 bug.duplicate_of = master
00684 
00685             # cache tags of master bug report instead of performing multiple
00686             # queries
00687             master_tags = master.tags
00688 
00689             if len(master.duplicates) == 10:
00690                 if 'escalation_tag' in self.options and self.options['escalation_tag'] not in master_tags and self.options.get('escalated_tag', ' invalid ') not in master_tags:
00691                     master.tags = master_tags + [self.options['escalation_tag']]  # LP#254901 workaround
00692                     master.lp_save()
00693 
00694                 if 'escalation_subscription' in self.options and self.options.get('escalated_tag', ' invalid ') not in master_tags:
00695                     p = self.launchpad.people[self.options['escalation_subscription']]
00696                     master.subscribe(person=p)
00697 
00698             # requesting updated stack trace?
00699             if report.has_useful_stacktrace() and ('apport-request-retrace' in master_tags
00700                                                    or 'apport-failed-retrace' in master_tags):
00701                 self.update(master_id, report, 'Updated stack trace from duplicate bug %i' % id,
00702                             key_filter=['Stacktrace', 'ThreadStacktrace',
00703                                         'Package', 'Dependencies', 'ProcMaps', 'ProcCmdline'])
00704 
00705                 master = self.launchpad.bugs[master_id]
00706                 x = master.tags[:]  # LP#254901 workaround
00707                 try:
00708                     x.remove('apport-failed-retrace')
00709                 except ValueError:
00710                     pass
00711                 try:
00712                     x.remove('apport-request-retrace')
00713                 except ValueError:
00714                     pass
00715                 master.tags = x
00716                 try:
00717                     master.lp_save()
00718                 except HTTPError:
00719                     pass  # LP#336866 workaround
00720 
00721             # white list of tags to copy from duplicates bugs to the master
00722             tags_to_copy = ['bugpattern-needed', 'running-unity']
00723             for series in self.lp_distro.series:
00724                 if series.status not in ['Active Development',
00725                                          'Current Stable Release', 'Supported']:
00726                     continue
00727                 tags_to_copy.append(series.name)
00728             # copy tags over from the duplicate bug to the master bug
00729             dupe_tags = set(bug.tags)
00730             # reload master tags as they may have changed
00731             master_tags = master.tags
00732             missing_tags = dupe_tags.difference(master_tags)
00733 
00734             for tag in missing_tags:
00735                 if tag in tags_to_copy:
00736                     master_tags.append(tag)
00737 
00738             master.tags = master_tags
00739             master.lp_save()
00740 
00741         else:
00742             if bug.duplicate_of:
00743                 bug.duplicate_of = None
00744 
00745         if bug._dirty_attributes:  # LP#336866 workaround
00746             bug.lp_save()
00747 
00748     def mark_regression(self, id, master):
00749         '''Mark a crash id as reintroducing an earlier crash which is
00750         already marked as fixed (having ID 'master').'''
00751 
00752         bug = self.launchpad.bugs[id]
00753         bug.newMessage(content='This crash has the same stack trace characteristics as bug #%i. \
00754 However, the latter was already fixed in an earlier package version than the \
00755 one in this report. This might be a regression or because the problem is \
00756 in a dependent package.' % master,
00757                        subject='Possible regression detected')
00758         bug = self.launchpad.bugs[id]  # fresh bug object, LP#336866 workaround
00759         bug.tags = bug.tags + ['regression-retracer']  # LP#254901 workaround
00760         bug.lp_save()
00761 
00762     def mark_retraced(self, id):
00763         '''Mark crash id as retraced.'''
00764 
00765         bug = self.launchpad.bugs[id]
00766         if self.arch_tag in bug.tags:
00767             x = bug.tags[:]  # LP#254901 workaround
00768             x.remove(self.arch_tag)
00769             bug.tags = x
00770             try:
00771                 bug.lp_save()
00772             except HTTPError:
00773                 pass  # LP#336866 workaround
00774 
00775     def mark_retrace_failed(self, id, invalid_msg=None):
00776         '''Mark crash id as 'failed to retrace'.'''
00777 
00778         bug = self.launchpad.bugs[id]
00779         if invalid_msg:
00780             try:
00781                 task = self._get_distro_tasks(bug.bug_tasks).next()
00782             except StopIteration:
00783                 # no distro task, just use the first one
00784                 task = bug.bug_tasks[0]
00785             task.status = 'Invalid'
00786             task.lp_save()
00787             bug.newMessage(content=invalid_msg,
00788                            subject='Crash report cannot be processed')
00789 
00790             for a in bug.attachments:
00791                 if a.title == 'CoreDump.gz':
00792                     try:
00793                         a.removeFromBug()
00794                     except HTTPError:
00795                         pass  # LP#249950 workaround
00796         else:
00797             if 'apport-failed-retrace' not in bug.tags:
00798                 bug.tags = bug.tags + ['apport-failed-retrace']  # LP#254901 workaround
00799                 bug.lp_save()
00800 
00801     def _mark_dup_checked(self, id, report):
00802         '''Mark crash id as checked for being a duplicate.'''
00803 
00804         bug = self.launchpad.bugs[id]
00805 
00806         # if we have a distro task without a package, fix it
00807         if 'SourcePackage' in report:
00808             for task in bug.bug_tasks:
00809                 if task.target.resource_type_link.endswith('#distribution'):
00810                     task.target = self.lp_distro.getSourcePackage(
00811                         name=report['SourcePackage'])
00812                     task.lp_save()
00813                     bug = self.launchpad.bugs[id]
00814                     break
00815 
00816         if 'need-duplicate-check' in bug.tags:
00817             x = bug.tags[:]  # LP#254901 workaround
00818             x.remove('need-duplicate-check')
00819             bug.tags = x
00820             bug.lp_save()
00821             if 'Traceback' in report:
00822                 for task in bug.bug_tasks:
00823                     if '#distribution' in task.target.resource_type_link:
00824                         if task.importance == 'Undecided':
00825                             task.importance = 'Medium'
00826                             task.lp_save()
00827         self._subscribe_triaging_team(bug, report)
00828 
00829     def known(self, report):
00830         '''Check if the crash db already knows about the crash signature.
00831 
00832         Check if the report has a DuplicateSignature, crash_signature(), or
00833         StacktraceAddressSignature, and ask the database whether the problem is
00834         already known. If so, return an URL where the user can check the status
00835         or subscribe (if available), or just return True if the report is known
00836         but there is no public URL. In that case the report will not be
00837         uploaded (i. e. upload() will not be called).
00838 
00839         Return None if the report does not have any signature or the crash
00840         database does not support checking for duplicates on the client side.
00841 
00842         The default implementation uses a text file format generated by
00843         duplicate_db_publish() at an URL specified by the "dupdb_url" option.
00844         Subclasses are free to override this with a custom implementation, such
00845         as a real database lookup.
00846         '''
00847         # we override the method here to check if the user actually has access
00848         # to the bug, and if the bug requests more retraces; in either case we
00849         # should file it.
00850         url = apport.crashdb.CrashDatabase.known(self, report)
00851 
00852         if not url:
00853             return url
00854 
00855         # record the fact that it is a duplicate, for triagers
00856         report['DuplicateOf'] = url
00857 
00858         try:
00859             f = urlopen(url + '/+text')
00860         except IOError:
00861             # if we are offline, or LP is down, upload will fail anyway, so we
00862             # can just as well avoid the upload
00863             return url
00864 
00865         line = f.readline()
00866         if not line.startswith(b'bug:'):
00867             # presumably a 404 etc. page, which happens for private bugs
00868             return True
00869 
00870         # check tags
00871         for line in f:
00872             if line.startswith(b'tags:'):
00873                 if b'apport-failed-retrace' in line or b'apport-request-retrace' in line:
00874                     return None
00875                 else:
00876                     break
00877 
00878             # stop at the first task, tags are in the first block
00879             if not line.strip():
00880                 break
00881 
00882         return url
00883 
00884     def _subscribe_triaging_team(self, bug, report):
00885         '''Subscribe the right triaging team to the bug.'''
00886 
00887         #FIXME: this entire function is an ugly Ubuntu specific hack until LP
00888         #gets a real crash db; see https://wiki.ubuntu.com/CrashReporting
00889 
00890         if 'DistroRelease' in report and report['DistroRelease'].split()[0] != 'Ubuntu':
00891             return  # only Ubuntu bugs are filed private
00892 
00893         #use a url hack here, it is faster
00894         person = '%s~%s' % (self.launchpad._root_uri,
00895                             self.options.get('triaging_team', 'ubuntu-crashes-universe'))
00896         bug.subscribe(person=person)
00897 
00898     def _generate_upload_blob(self, report):
00899         '''Generate a multipart/MIME temporary file for uploading.
00900 
00901         You have to close the returned file object after you are done with it.
00902         '''
00903         # set reprocessing tags
00904         hdr = {}
00905         hdr['Tags'] = 'apport-%s' % report['ProblemType'].lower()
00906         a = report.get('PackageArchitecture')
00907         if not a or a == 'all':
00908             a = report.get('Architecture')
00909         if a:
00910             hdr['Tags'] += ' ' + a
00911         if 'Tags' in report:
00912             hdr['Tags'] += ' ' + report['Tags'].lower()
00913 
00914         # privacy/retracing for distro reports
00915         # FIXME: ugly hack until LP has a real crash db
00916         if 'DistroRelease' in report:
00917             if a and ('VmCore' in report or 'CoreDump' in report):
00918                 hdr['Private'] = 'yes'
00919                 hdr['Subscribers'] = self.options.get('initial_subscriber', 'apport')
00920                 hdr['Tags'] += ' need-%s-retrace' % a
00921             elif 'Traceback' in report:
00922                 hdr['Private'] = 'yes'
00923                 hdr['Subscribers'] = 'apport'
00924                 hdr['Tags'] += ' need-duplicate-check'
00925         if 'DuplicateSignature' in report and 'need-duplicate-check' not in hdr['Tags']:
00926                 hdr['Tags'] += ' need-duplicate-check'
00927 
00928         # if we have checkbox submission key, link it to the bug; keep text
00929         # reference until the link is shown in Launchpad's UI
00930         if 'CheckboxSubmission' in report:
00931             hdr['HWDB-Submission'] = report['CheckboxSubmission']
00932 
00933         # order in which keys should appear in the temporary file
00934         order = ['ProblemType', 'DistroRelease', 'Package', 'Regression', 'Reproducible',
00935                  'TestedUpstream', 'ProcVersionSignature', 'Uname', 'NonfreeKernelModules']
00936 
00937         # write MIME/Multipart version into temporary file
00938         mime = tempfile.TemporaryFile()
00939         report.write_mime(mime, extra_headers=hdr, skip_keys=['Tags'], priority_fields=order)
00940         mime.flush()
00941         mime.seek(0)
00942 
00943         return mime
00944 
00945 #
00946 # Launchpad storeblob API (should go into launchpadlib, see LP #315358)
00947 #
00948 
00949 _https_upload_callback = None
00950 
00951 
00952 #
00953 # This progress code is based on KodakLoader by Jason Hildebrand
00954 # <jason@opensky.ca>. See http://www.opensky.ca/~jdhildeb/software/kodakloader/
00955 # for details.
00956 class HTTPSProgressConnection(HTTPSConnection):
00957     '''Implement a HTTPSConnection with an optional callback function for
00958     upload progress.'''
00959 
00960     def send(self, data):
00961         global _https_upload_callback
00962 
00963         # if callback has not been set, call the old method
00964         if not _https_upload_callback:
00965             HTTPSConnection.send(self, data)
00966             return
00967 
00968         sent = 0
00969         total = len(data)
00970         chunksize = 1024
00971         while sent < total:
00972             _https_upload_callback(sent, total)
00973             t1 = time.time()
00974             HTTPSConnection.send(self, data[sent:(sent + chunksize)])
00975             sent += chunksize
00976             t2 = time.time()
00977 
00978             # adjust chunksize so that it takes between .5 and 2
00979             # seconds to send a chunk
00980             if chunksize > 1024:
00981                 if t2 - t1 < .5:
00982                     chunksize <<= 1
00983                 elif t2 - t1 > 2:
00984                     chunksize >>= 1
00985 
00986 
00987 class HTTPSProgressHandler(HTTPSHandler):
00988 
00989     def https_open(self, req):
00990         return self.do_open(HTTPSProgressConnection, req)
00991 
00992 
00993 def upload_blob(blob, progress_callback=None, hostname='launchpad.net'):
00994     '''Upload blob (file-like object) to Launchpad.
00995 
00996     progress_callback can be set to a function(sent, total) which is regularly
00997     called with the number of bytes already sent and total number of bytes to
00998     send. It is called every 0.5 to 2 seconds (dynamically adapted to upload
00999     bandwidth).
01000 
01001     Return None on error, or the ticket number on success.
01002 
01003     By default this uses the production Launchpad hostname. Set
01004     hostname to 'launchpad.dev' or 'staging.launchpad.net' to use another
01005     instance for testing.
01006     '''
01007     ticket = None
01008     url = 'https://%s/+storeblob' % hostname
01009 
01010     global _https_upload_callback
01011     _https_upload_callback = progress_callback
01012 
01013     # build the form-data multipart/MIME request
01014     data = email.mime.multipart.MIMEMultipart()
01015 
01016     submit = email.mime.text.MIMEText('1')
01017     submit.add_header('Content-Disposition', 'form-data; name="FORM_SUBMIT"')
01018     data.attach(submit)
01019 
01020     form_blob = email.mime.base.MIMEBase('application', 'octet-stream')
01021     form_blob.add_header('Content-Disposition', 'form-data; name="field.blob"; filename="x"')
01022     form_blob.set_payload(blob.read().decode('ascii'))
01023     data.attach(form_blob)
01024 
01025     data_flat = BytesIO()
01026     if sys.version_info.major == 2:
01027         gen = email.generator.Generator(data_flat, mangle_from_=False)
01028     else:
01029         gen = email.generator.BytesGenerator(data_flat, mangle_from_=False)
01030     gen.flatten(data)
01031 
01032     # do the request; we need to explicitly set the content type here, as it
01033     # defaults to x-www-form-urlencoded
01034     req = Request(url, data_flat.getvalue())
01035     req.add_header('Content-Type', 'multipart/form-data; boundary=' + data.get_boundary())
01036     opener = build_opener(HTTPSProgressHandler)
01037     result = opener.open(req)
01038     ticket = result.info().get('X-Launchpad-Blob-Token')
01039 
01040     assert ticket
01041     return ticket
01042 
01043 #
01044 # Unit tests
01045 #
01046 
01047 if __name__ == '__main__':
01048     import unittest, atexit, shutil, subprocess
01049     import mock
01050 
01051     crashdb = None
01052     _segv_report = None
01053     _python_report = None
01054     _uncommon_description_report = None
01055 
01056     class _T(unittest.TestCase):
01057         # this assumes that a source package 'coreutils' exists and builds a
01058         # binary package 'coreutils'
01059         test_package = 'coreutils'
01060         test_srcpackage = 'coreutils'
01061 
01062         #
01063         # Generic tests, should work for all CrashDB implementations
01064         #
01065 
01066         def setUp(self):
01067             global crashdb
01068             if not crashdb:
01069                 crashdb = self._get_instance()
01070             self.crashdb = crashdb
01071 
01072             # create a local reference report so that we can compare
01073             # DistroRelease, Architecture, etc.
01074             self.ref_report = apport.Report()
01075             self.ref_report.add_os_info()
01076             self.ref_report.add_user_info()
01077             self.ref_report['SourcePackage'] = 'coreutils'
01078 
01079             # Objects tests rely on.
01080             self._create_project('langpack-o-matic')
01081 
01082         def _create_project(self, name):
01083             '''Create a project using launchpadlib to be used by tests.'''
01084 
01085             project = self.crashdb.launchpad.projects[name]
01086             if not project:
01087                 self.crashdb.launchpad.projects.new_project(
01088                     description=name + 'description',
01089                     display_name=name,
01090                     name=name,
01091                     summary=name + 'summary',
01092                     title=name + 'title')
01093 
01094         @property
01095         def hostname(self):
01096             '''Get the Launchpad hostname for the given crashdb.'''
01097 
01098             return self.crashdb.get_hostname()
01099 
01100         def get_segv_report(self, force_fresh=False):
01101             '''Generate SEGV crash report.
01102 
01103             This is only done once, subsequent calls will return the already
01104             existing ID, unless force_fresh is True.
01105 
01106             Return the ID.
01107             '''
01108             global _segv_report
01109             if not force_fresh and _segv_report is not None:
01110                 return _segv_report
01111 
01112             r = self._generate_sigsegv_report()
01113             r.add_package_info(self.test_package)
01114             r.add_os_info()
01115             r.add_gdb_info()
01116             r.add_user_info()
01117             self.assertEqual(r.standard_title(), 'crash crashed with SIGSEGV in f()')
01118 
01119             # add some binary gibberish which isn't UTF-8
01120             r['ShortGibberish'] = ' "]\xb6"\n'
01121             r['LongGibberish'] = 'a\nb\nc\nd\ne\n\xff\xff\xff\n\f'
01122 
01123             # create a bug for the report
01124             bug_target = self._get_bug_target(self.crashdb, r)
01125             self.assertTrue(bug_target)
01126 
01127             id = self._file_bug(bug_target, r)
01128             self.assertTrue(id > 0)
01129 
01130             sys.stderr.write('(Created SEGV report: https://%s/bugs/%i) ' % (self.hostname, id))
01131             if not force_fresh:
01132                 _segv_report = id
01133             return id
01134 
01135         def get_python_report(self):
01136             '''Generate Python crash report.
01137 
01138             Return the ID.
01139             '''
01140             global _python_report
01141             if _python_report is not None:
01142                 return _python_report
01143 
01144             r = apport.Report('Crash')
01145             r['ExecutablePath'] = '/bin/foo'
01146             r['Traceback'] = '''Traceback (most recent call last):
01147   File "/bin/foo", line 67, in fuzz
01148     print(weird)
01149 NameError: global name 'weird' is not defined'''
01150             r['Tags'] = 'boogus pybogus'
01151             r.add_package_info(self.test_package)
01152             r.add_os_info()
01153             r.add_user_info()
01154             self.assertEqual(r.standard_title(),
01155                              "foo crashed with NameError in fuzz(): global name 'weird' is not defined")
01156 
01157             bug_target = self._get_bug_target(self.crashdb, r)
01158             self.assertTrue(bug_target)
01159 
01160             id = self._file_bug(bug_target, r)
01161             self.assertTrue(id > 0)
01162             sys.stderr.write('(Created Python report: https://%s/bugs/%i) ' % (self.hostname, id))
01163             _python_report = id
01164             return id
01165 
01166         def get_uncommon_description_report(self, force_fresh=False):
01167             '''File a bug report with an uncommon description.
01168 
01169             This is only done once, subsequent calls will return the already
01170             existing ID, unless force_fresh is True.
01171 
01172             Example taken from real LP bug 269539. It contains only
01173             ProblemType/Architecture/DistroRelease in the description, and has
01174             free-form description text after the Apport data.
01175 
01176             Return the ID.
01177             '''
01178             global _uncommon_description_report
01179             if not force_fresh and _uncommon_description_report is not None:
01180                 return _uncommon_description_report
01181 
01182             desc = '''problem
01183 
01184 ProblemType: Package
01185 Architecture: amd64
01186 DistroRelease: Ubuntu 8.10
01187 
01188 more text
01189 
01190 and more
01191 '''
01192             bug = self.crashdb.launchpad.bugs.createBug(
01193                 title=b'mixed description bug'.encode(),
01194                 description=desc,
01195                 target=self.crashdb.lp_distro)
01196             sys.stderr.write('(Created uncommon description: https://%s/bugs/%i) ' % (self.hostname, bug.id))
01197 
01198             if not force_fresh:
01199                 _uncommon_description_report = bug.id
01200             return bug.id
01201 
01202         def test_1_download(self):
01203             '''download()'''
01204 
01205             r = self.crashdb.download(self.get_segv_report())
01206             self.assertEqual(r['ProblemType'], 'Crash')
01207             self.assertEqual(r['Title'], 'crash crashed with SIGSEGV in f()')
01208             self.assertEqual(r['DistroRelease'], self.ref_report['DistroRelease'])
01209             self.assertEqual(r['Architecture'], self.ref_report['Architecture'])
01210             self.assertEqual(r['Uname'], self.ref_report['Uname'])
01211             self.assertEqual(r.get('NonfreeKernelModules'),
01212                              self.ref_report.get('NonfreeKernelModules'))
01213             self.assertEqual(r.get('UserGroups'), self.ref_report.get('UserGroups'))
01214             tags = set(r['Tags'].split())
01215             self.assertEqual(tags, set([self.crashdb.arch_tag, 'apport-crash',
01216                                         apport.packaging.get_system_architecture()]))
01217 
01218             self.assertEqual(r['Signal'], '11')
01219             self.assertTrue(r['ExecutablePath'].endswith('/crash'))
01220             self.assertEqual(r['SourcePackage'], self.test_srcpackage)
01221             self.assertTrue(r['Package'].startswith(self.test_package + ' '))
01222             self.assertTrue('f (x=42)' in r['Stacktrace'])
01223             self.assertTrue('f (x=42)' in r['StacktraceTop'])
01224             self.assertTrue('f (x=42)' in r['ThreadStacktrace'])
01225             self.assertTrue(len(r['CoreDump']) > 1000)
01226             self.assertTrue('Dependencies' in r)
01227             self.assertTrue('Disassembly' in r)
01228             self.assertTrue('Registers' in r)
01229 
01230             # check tags
01231             r = self.crashdb.download(self.get_python_report())
01232             tags = set(r['Tags'].split())
01233             self.assertEqual(tags, set(['apport-crash', 'boogus', 'pybogus',
01234                                         'need-duplicate-check', apport.packaging.get_system_architecture()]))
01235 
01236         def test_2_update_traces(self):
01237             '''update_traces()'''
01238 
01239             r = self.crashdb.download(self.get_segv_report())
01240             self.assertTrue('CoreDump' in r)
01241             self.assertTrue('Dependencies' in r)
01242             self.assertTrue('Disassembly' in r)
01243             self.assertTrue('Registers' in r)
01244             self.assertTrue('Stacktrace' in r)
01245             self.assertTrue('ThreadStacktrace' in r)
01246             self.assertEqual(r['Title'], 'crash crashed with SIGSEGV in f()')
01247 
01248             # updating with a useless stack trace retains core dump
01249             r['StacktraceTop'] = '?? ()'
01250             r['Stacktrace'] = 'long\ntrace'
01251             r['ThreadStacktrace'] = 'thread\neven longer\ntrace'
01252             r['FooBar'] = 'bogus'
01253             self.crashdb.update_traces(self.get_segv_report(), r, 'I can has a better retrace?')
01254             r = self.crashdb.download(self.get_segv_report())
01255             self.assertTrue('CoreDump' in r)
01256             self.assertTrue('Dependencies' in r)
01257             self.assertTrue('Disassembly' in r)
01258             self.assertTrue('Registers' in r)
01259             self.assertTrue('Stacktrace' in r)  # TODO: ascertain that it's the updated one
01260             self.assertTrue('ThreadStacktrace' in r)
01261             self.assertFalse('FooBar' in r)
01262             self.assertEqual(r['Title'], 'crash crashed with SIGSEGV in f()')
01263 
01264             tags = self.crashdb.launchpad.bugs[self.get_segv_report()].tags
01265             self.assertTrue('apport-crash' in tags)
01266             self.assertFalse('apport-collected' in tags)
01267 
01268             # updating with a useful stack trace removes core dump
01269             r['StacktraceTop'] = 'read () from /lib/libc.6.so\nfoo (i=1) from /usr/lib/libfoo.so'
01270             r['Stacktrace'] = 'long\ntrace'
01271             r['ThreadStacktrace'] = 'thread\neven longer\ntrace'
01272             self.crashdb.update_traces(self.get_segv_report(), r, 'good retrace!')
01273             r = self.crashdb.download(self.get_segv_report())
01274             self.assertFalse('CoreDump' in r)
01275             self.assertTrue('Dependencies' in r)
01276             self.assertTrue('Disassembly' in r)
01277             self.assertTrue('Registers' in r)
01278             self.assertTrue('Stacktrace' in r)
01279             self.assertTrue('ThreadStacktrace' in r)
01280             self.assertFalse('FooBar' in r)
01281 
01282             # as previous title had standard form, the top function gets
01283             # updated
01284             self.assertEqual(r['Title'], 'crash crashed with SIGSEGV in read()')
01285 
01286             # respects title amendments
01287             bug = self.crashdb.launchpad.bugs[self.get_segv_report()]
01288             bug.title = 'crash crashed with SIGSEGV in f() on exit'
01289             try:
01290                 bug.lp_save()
01291             except HTTPError:
01292                 pass  # LP#336866 workaround
01293             r['StacktraceTop'] = 'read () from /lib/libc.6.so\nfoo (i=1) from /usr/lib/libfoo.so'
01294             self.crashdb.update_traces(self.get_segv_report(), r, 'good retrace with title amendment')
01295             r = self.crashdb.download(self.get_segv_report())
01296             self.assertEqual(r['Title'], 'crash crashed with SIGSEGV in read() on exit')
01297 
01298             # does not destroy custom titles
01299             bug = self.crashdb.launchpad.bugs[self.get_segv_report()]
01300             bug.title = 'crash is crashy'
01301             try:
01302                 bug.lp_save()
01303             except HTTPError:
01304                 pass  # LP#336866 workaround
01305 
01306             r['StacktraceTop'] = 'read () from /lib/libc.6.so\nfoo (i=1) from /usr/lib/libfoo.so'
01307             self.crashdb.update_traces(self.get_segv_report(), r, 'good retrace with custom title')
01308             r = self.crashdb.download(self.get_segv_report())
01309             self.assertEqual(r['Title'], 'crash is crashy')
01310 
01311             # test various situations which caused crashes
01312             r['Stacktrace'] = ''  # empty file
01313             r['ThreadStacktrace'] = '"]\xb6"\n'  # not interpretable as UTF-8, LP #353805
01314             r['StacktraceSource'] = 'a\nb\nc\nd\ne\n\xff\xff\xff\n\f'
01315             self.crashdb.update_traces(self.get_segv_report(), r, 'tests')
01316 
01317         def test_get_comment_url(self):
01318             '''get_comment_url() for non-ASCII titles'''
01319 
01320             # UTF-8 bytestring, works in both python 2.7 and 3
01321             title = b'1\xc3\xa4\xe2\x99\xa52'
01322 
01323             # distro, UTF-8 bytestring
01324             r = apport.Report('Bug')
01325             r['Title'] = title
01326             url = self.crashdb.get_comment_url(r, 42)
01327             self.assertTrue(url.endswith('/ubuntu/+filebug/42?field.title=1%C3%A4%E2%99%A52'))
01328 
01329             # distro, unicode
01330             r['Title'] = title.decode('UTF-8')
01331             url = self.crashdb.get_comment_url(r, 42)
01332             self.assertTrue(url.endswith('/ubuntu/+filebug/42?field.title=1%C3%A4%E2%99%A52'))
01333 
01334             # package, unicode
01335             r['SourcePackage'] = 'coreutils'
01336             url = self.crashdb.get_comment_url(r, 42)
01337             self.assertTrue(url.endswith('/ubuntu/+source/coreutils/+filebug/42?field.title=1%C3%A4%E2%99%A52'))
01338 
01339         def test_update_description(self):
01340             '''update() with changing description'''
01341 
01342             bug_target = self.crashdb.lp_distro.getSourcePackage(name='bash')
01343             bug = self.crashdb.launchpad.bugs.createBug(
01344                 description='test description for test bug.',
01345                 target=bug_target,
01346                 title='testbug')
01347             id = bug.id
01348             self.assertTrue(id > 0)
01349             sys.stderr.write('(https://%s/bugs/%i) ' % (self.hostname, id))
01350 
01351             r = apport.Report('Bug')
01352 
01353             r['OneLiner'] = b'bogus\xe2\x86\x92'.decode('UTF-8')
01354             r['StacktraceTop'] = 'f()\ng()\nh(1)'
01355             r['ShortGoo'] = 'lineone\nlinetwo'
01356             r['DpkgTerminalLog'] = 'one\ntwo\nthree\nfour\nfive\nsix'
01357             r['VarLogDistupgradeBinGoo'] = b'\x01' * 1024
01358 
01359             self.crashdb.update(id, r, 'NotMe', change_description=True)
01360 
01361             r = self.crashdb.download(id)
01362 
01363             self.assertEqual(r['OneLiner'], b'bogus\xe2\x86\x92'.decode('UTF-8'))
01364             self.assertEqual(r['ShortGoo'], 'lineone\nlinetwo')
01365             self.assertEqual(r['DpkgTerminalLog'], 'one\ntwo\nthree\nfour\nfive\nsix')
01366             self.assertEqual(r['VarLogDistupgradeBinGoo'], b'\x01' * 1024)
01367 
01368             self.assertEqual(self.crashdb.launchpad.bugs[id].tags,
01369                              ['apport-collected'])
01370 
01371         def test_update_comment(self):
01372             '''update() with appending comment'''
01373 
01374             bug_target = self.crashdb.lp_distro.getSourcePackage(name='bash')
01375             # we need to fake an apport description separator here, since we
01376             # want to be lazy and use download() for checking the result
01377             bug = self.crashdb.launchpad.bugs.createBug(
01378                 description='Pr0blem\n\n--- \nProblemType: Bug',
01379                 target=bug_target,
01380                 title='testbug')
01381             id = bug.id
01382             self.assertTrue(id > 0)
01383             sys.stderr.write('(https://%s/bugs/%i) ' % (self.hostname, id))
01384 
01385             r = apport.Report('Bug')
01386 
01387             r['OneLiner'] = 'bogus→'
01388             r['StacktraceTop'] = 'f()\ng()\nh(1)'
01389             r['ShortGoo'] = 'lineone\nlinetwo'
01390             r['DpkgTerminalLog'] = 'one\ntwo\nthree\nfour\nfive\nsix'
01391             r['VarLogDistupgradeBinGoo'] = '\x01' * 1024
01392 
01393             self.crashdb.update(id, r, 'meow', change_description=False)
01394 
01395             r = self.crashdb.download(id)
01396 
01397             self.assertFalse('OneLiner' in r)
01398             self.assertFalse('ShortGoo' in r)
01399             self.assertEqual(r['ProblemType'], 'Bug')
01400             self.assertEqual(r['DpkgTerminalLog'], 'one\ntwo\nthree\nfour\nfive\nsix')
01401             self.assertEqual(r['VarLogDistupgradeBinGoo'], '\x01' * 1024)
01402 
01403             self.assertEqual(self.crashdb.launchpad.bugs[id].tags,
01404                              ['apport-collected'])
01405 
01406         def test_update_filter(self):
01407             '''update() with a key filter'''
01408 
01409             bug_target = self.crashdb.lp_distro.getSourcePackage(name='bash')
01410             bug = self.crashdb.launchpad.bugs.createBug(
01411                 description='test description for test bug',
01412                 target=bug_target,
01413                 title='testbug')
01414             id = bug.id
01415             self.assertTrue(id > 0)
01416             sys.stderr.write('(https://%s/bugs/%i) ' % (self.hostname, id))
01417 
01418             r = apport.Report('Bug')
01419 
01420             r['OneLiner'] = 'bogus→'
01421             r['StacktraceTop'] = 'f()\ng()\nh(1)'
01422             r['ShortGoo'] = 'lineone\nlinetwo'
01423             r['DpkgTerminalLog'] = 'one\ntwo\nthree\nfour\nfive\nsix'
01424             r['VarLogDistupgradeBinGoo'] = '\x01' * 1024
01425 
01426             self.crashdb.update(id, r, 'NotMe', change_description=True,
01427                                 key_filter=['ProblemType', 'ShortGoo', 'DpkgTerminalLog'])
01428 
01429             r = self.crashdb.download(id)
01430 
01431             self.assertFalse('OneLiner' in r)
01432             self.assertEqual(r['ShortGoo'], 'lineone\nlinetwo')
01433             self.assertEqual(r['ProblemType'], 'Bug')
01434             self.assertEqual(r['DpkgTerminalLog'], 'one\ntwo\nthree\nfour\nfive\nsix')
01435             self.assertFalse('VarLogDistupgradeBinGoo' in r)
01436 
01437             self.assertEqual(self.crashdb.launchpad.bugs[id].tags, [])
01438 
01439         def test_get_distro_release(self):
01440             '''get_distro_release()'''
01441 
01442             self.assertEqual(self.crashdb.get_distro_release(self.get_segv_report()),
01443                              self.ref_report['DistroRelease'])
01444 
01445         def test_get_affected_packages(self):
01446             '''get_affected_packages()'''
01447 
01448             self.assertEqual(self.crashdb.get_affected_packages(self.get_segv_report()),
01449                              [self.ref_report['SourcePackage']])
01450 
01451         def test_is_reporter(self):
01452             '''is_reporter()'''
01453 
01454             self.assertTrue(self.crashdb.is_reporter(self.get_segv_report()))
01455             self.assertFalse(self.crashdb.is_reporter(1))
01456 
01457         def test_can_update(self):
01458             '''can_update()'''
01459 
01460             self.assertTrue(self.crashdb.can_update(self.get_segv_report()))
01461             self.assertFalse(self.crashdb.can_update(1))
01462 
01463         def test_duplicates(self):
01464             '''duplicate handling'''
01465 
01466             # initially we have no dups
01467             self.assertEqual(self.crashdb.duplicate_of(self.get_segv_report()), None)
01468             self.assertEqual(self.crashdb.get_fixed_version(self.get_segv_report()), None)
01469 
01470             segv_id = self.get_segv_report()
01471             known_test_id = self.get_uncommon_description_report()
01472             known_test_id2 = self.get_uncommon_description_report(force_fresh=True)
01473 
01474             # dupe our segv_report and check that it worked; then undupe it
01475             r = self.crashdb.download(segv_id)
01476             self.crashdb.close_duplicate(r, segv_id, known_test_id)
01477             self.assertEqual(self.crashdb.duplicate_of(segv_id), known_test_id)
01478 
01479             # this should be a no-op
01480             self.crashdb.close_duplicate(r, segv_id, known_test_id)
01481             self.assertEqual(self.crashdb.duplicate_of(segv_id), known_test_id)
01482 
01483             self.assertEqual(self.crashdb.get_fixed_version(segv_id), 'invalid')
01484             self.crashdb.close_duplicate(r, segv_id, None)
01485             self.assertEqual(self.crashdb.duplicate_of(segv_id), None)
01486             self.assertEqual(self.crashdb.get_fixed_version(segv_id), None)
01487 
01488             # this should have removed attachments; note that Stacktrace is
01489             # short, and thus inline
01490             r = self.crashdb.download(self.get_segv_report())
01491             self.assertFalse('CoreDump' in r)
01492             self.assertFalse('Disassembly' in r)
01493             self.assertFalse('ProcMaps' in r)
01494             self.assertFalse('ProcStatus' in r)
01495             self.assertFalse('Registers' in r)
01496             self.assertFalse('ThreadStacktrace' in r)
01497 
01498             # now try duplicating to a duplicate bug; this should automatically
01499             # transition to the master bug
01500             self.crashdb.close_duplicate(apport.Report(), known_test_id,
01501                                          known_test_id2)
01502             self.crashdb.close_duplicate(r, segv_id, known_test_id)
01503             self.assertEqual(self.crashdb.duplicate_of(segv_id),
01504                              known_test_id2)
01505 
01506             self.crashdb.close_duplicate(apport.Report(), known_test_id, None)
01507             self.crashdb.close_duplicate(apport.Report(), known_test_id2, None)
01508             self.crashdb.close_duplicate(r, segv_id, None)
01509 
01510             # this should be a no-op
01511             self.crashdb.close_duplicate(apport.Report(), known_test_id, None)
01512             self.assertEqual(self.crashdb.duplicate_of(known_test_id), None)
01513 
01514             self.crashdb.mark_regression(segv_id, known_test_id)
01515             self._verify_marked_regression(segv_id)
01516 
01517         def test_marking_segv(self):
01518             '''processing status markings for signal crashes'''
01519 
01520             # mark_retraced()
01521             unretraced_before = self.crashdb.get_unretraced()
01522             self.assertTrue(self.get_segv_report() in unretraced_before)
01523             self.assertFalse(self.get_python_report() in unretraced_before)
01524             self.crashdb.mark_retraced(self.get_segv_report())
01525             unretraced_after = self.crashdb.get_unretraced()
01526             self.assertFalse(self.get_segv_report() in unretraced_after)
01527             self.assertEqual(unretraced_before,
01528                              unretraced_after.union(set([self.get_segv_report()])))
01529             self.assertEqual(self.crashdb.get_fixed_version(self.get_segv_report()), None)
01530 
01531             # mark_retrace_failed()
01532             self._mark_needs_retrace(self.get_segv_report())
01533             self.crashdb.mark_retraced(self.get_segv_report())
01534             self.crashdb.mark_retrace_failed(self.get_segv_report())
01535             unretraced_after = self.crashdb.get_unretraced()
01536             self.assertFalse(self.get_segv_report() in unretraced_after)
01537             self.assertEqual(unretraced_before,
01538                              unretraced_after.union(set([self.get_segv_report()])))
01539             self.assertEqual(self.crashdb.get_fixed_version(self.get_segv_report()), None)
01540 
01541             # mark_retrace_failed() of invalid bug
01542             self._mark_needs_retrace(self.get_segv_report())
01543             self.crashdb.mark_retraced(self.get_segv_report())
01544             self.crashdb.mark_retrace_failed(self.get_segv_report(), "I don't like you")
01545             unretraced_after = self.crashdb.get_unretraced()
01546             self.assertFalse(self.get_segv_report() in unretraced_after)
01547             self.assertEqual(unretraced_before,
01548                              unretraced_after.union(set([self.get_segv_report()])))
01549             self.assertEqual(self.crashdb.get_fixed_version(self.get_segv_report()),
01550                              'invalid')
01551 
01552         def test_marking_project(self):
01553             '''processing status markings for a project CrashDB'''
01554 
01555             # create a distro bug
01556             distro_bug = self.crashdb.launchpad.bugs.createBug(
01557                 description='foo',
01558                 tags=self.crashdb.arch_tag,
01559                 target=self.crashdb.lp_distro,
01560                 title='ubuntu distro retrace bug')
01561             #print('distro bug: https://staging.launchpad.net/bugs/%i' % distro_bug.id)
01562 
01563             # create a project crash DB and a bug
01564             launchpad_instance = os.environ.get('APPORT_LAUNCHPAD_INSTANCE') or 'staging'
01565 
01566             project_db = CrashDatabase(
01567                 os.environ.get('LP_CREDENTIALS'),
01568                 {'project': 'langpack-o-matic', 'launchpad_instance': launchpad_instance})
01569             project_bug = project_db.launchpad.bugs.createBug(
01570                 description='bar',
01571                 tags=project_db.arch_tag,
01572                 target=project_db.lp_distro,
01573                 title='project retrace bug')
01574             #print('project bug: https://staging.launchpad.net/bugs/%i' % project_bug.id)
01575 
01576             # on project_db, we recognize the project bug and can mark it
01577             unretraced_before = project_db.get_unretraced()
01578             self.assertTrue(project_bug.id in unretraced_before)
01579             self.assertFalse(distro_bug.id in unretraced_before)
01580             project_db.mark_retraced(project_bug.id)
01581             unretraced_after = project_db.get_unretraced()
01582             self.assertFalse(project_bug.id in unretraced_after)
01583             self.assertEqual(unretraced_before,
01584                              unretraced_after.union(set([project_bug.id])))
01585             self.assertEqual(self.crashdb.get_fixed_version(project_bug.id), None)
01586 
01587         def test_marking_python(self):
01588             '''processing status markings for interpreter crashes'''
01589 
01590             unchecked_before = self.crashdb.get_dup_unchecked()
01591             self.assertTrue(self.get_python_report() in unchecked_before)
01592             self.assertFalse(self.get_segv_report() in unchecked_before)
01593             self.crashdb._mark_dup_checked(self.get_python_report(), self.ref_report)
01594             unchecked_after = self.crashdb.get_dup_unchecked()
01595             self.assertFalse(self.get_python_report() in unchecked_after)
01596             self.assertEqual(unchecked_before,
01597                              unchecked_after.union(set([self.get_python_report()])))
01598             self.assertEqual(self.crashdb.get_fixed_version(self.get_python_report()), None)
01599 
01600         def test_update_traces_invalid(self):
01601             '''updating an invalid crash
01602 
01603             This simulates a race condition where a crash being processed gets
01604             invalidated by marking it as a duplicate.
01605             '''
01606             id = self.get_segv_report(force_fresh=True)
01607 
01608             r = self.crashdb.download(id)
01609 
01610             self.crashdb.close_duplicate(r, id, self.get_segv_report())
01611 
01612             # updating with a useful stack trace removes core dump
01613             r['StacktraceTop'] = 'read () from /lib/libc.6.so\nfoo (i=1) from /usr/lib/libfoo.so'
01614             r['Stacktrace'] = 'long\ntrace'
01615             r['ThreadStacktrace'] = 'thread\neven longer\ntrace'
01616             self.crashdb.update_traces(id, r, 'good retrace!')
01617 
01618             r = self.crashdb.download(id)
01619             self.assertFalse('CoreDump' in r)
01620 
01621         @mock.patch.object(CrashDatabase, '_get_source_version')
01622         def test_get_fixed_version(self, *args):
01623             '''get_fixed_version() for fixed bugs
01624 
01625             Other cases are already checked in test_marking_segv() (invalid
01626             bugs) and test_duplicates (duplicate bugs) for efficiency.
01627             '''
01628             # staging.launchpad.net often does not have Quantal, so mock-patch
01629             # it to a known value
01630             CrashDatabase._get_source_version.return_value = '3.14'
01631             self._mark_report_fixed(self.get_segv_report())
01632             fixed_ver = self.crashdb.get_fixed_version(self.get_segv_report())
01633             self.assertEqual(fixed_ver, '3.14')
01634             self._mark_report_new(self.get_segv_report())
01635             self.assertEqual(self.crashdb.get_fixed_version(self.get_segv_report()), None)
01636 
01637         #
01638         # Launchpad specific implementation and tests
01639         #
01640 
01641         @classmethod
01642         def _get_instance(klass):
01643             '''Create a CrashDB instance'''
01644 
01645             launchpad_instance = os.environ.get('APPORT_LAUNCHPAD_INSTANCE') or 'staging'
01646 
01647             return CrashDatabase(os.environ.get('LP_CREDENTIALS'),
01648                                  {'distro': 'ubuntu',
01649                                   'launchpad_instance': launchpad_instance})
01650 
01651         def _get_bug_target(self, db, report):
01652             '''Return the bug_target for this report.'''
01653 
01654             project = db.options.get('project')
01655             if 'SourcePackage' in report:
01656                 return db.lp_distro.getSourcePackage(name=report['SourcePackage'])
01657             elif project:
01658                 return db.launchpad.projects[project]
01659             else:
01660                 return self.lp_distro
01661 
01662         def _file_bug(self, bug_target, report, description=None):
01663             '''File a bug report for a report.
01664 
01665             Return the bug ID.
01666             '''
01667             # unfortunately staging's +storeblob API hardly ever works, so we
01668             # must avoid using it. Fake it by manually doing the comments and
01669             # attachments that +filebug would ordinarily do itself when given a
01670             # blob handle.
01671 
01672             if description is None:
01673                 description = 'some description'
01674 
01675             mime = self.crashdb._generate_upload_blob(report)
01676             msg = email.message_from_file(mime)
01677             mime.close()
01678             msg_iter = msg.walk()
01679 
01680             # first one is the multipart container
01681             header = msg_iter.next()
01682             assert header.is_multipart()
01683 
01684             # second part should be an inline text/plain attachments with all short
01685             # fields
01686             part = msg_iter.next()
01687             assert not part.is_multipart()
01688             assert part.get_content_type() == 'text/plain'
01689             description += '\n\n' + part.get_payload(decode=True).decode('UTF-8', 'replace')
01690 
01691             # create the bug from header and description data
01692             bug = self.crashdb.launchpad.bugs.createBug(
01693                 description=description,
01694                 private=(header['Private'] == 'yes'),
01695                 tags=header['Tags'].split(),
01696                 target=bug_target,
01697                 title=report.get('Title', report.standard_title()))
01698 
01699             # nwo add the attachments
01700             for part in msg_iter:
01701                 assert not part.is_multipart()
01702                 bug.addAttachment(comment='',
01703                                   description=part.get_filename(),
01704                                   content_type=None,
01705                                   data=part.get_payload(decode=True),
01706                                   filename=part.get_filename(), is_patch=False)
01707 
01708             for subscriber in header['Subscribers'].split():
01709                 sub = self.crashdb.launchpad.people[subscriber]
01710                 if sub:
01711                     bug.subscribe(person=sub)
01712 
01713             return bug.id
01714 
01715         def _mark_needs_retrace(self, id):
01716             '''Mark a report ID as needing retrace.'''
01717 
01718             bug = self.crashdb.launchpad.bugs[id]
01719             if self.crashdb.arch_tag not in bug.tags:
01720                 bug.tags = bug.tags + [self.crashdb.arch_tag]
01721                 bug.lp_save()
01722 
01723         def _mark_needs_dupcheck(self, id):
01724             '''Mark a report ID as needing duplicate check.'''
01725 
01726             bug = self.crashdb.launchpad.bugs[id]
01727             if 'need-duplicate-check' not in bug.tags:
01728                 bug.tags = bug.tags + ['need-duplicate-check']
01729                 bug.lp_save()
01730 
01731         def _mark_report_fixed(self, id):
01732             '''Close a report ID as "fixed".'''
01733 
01734             bug = self.crashdb.launchpad.bugs[id]
01735             tasks = list(bug.bug_tasks)
01736             assert len(tasks) == 1
01737             t = tasks[0]
01738             t.status = 'Fix Released'
01739             t.lp_save()
01740 
01741         def _mark_report_new(self, id):
01742             '''Reopen a report ID as "new".'''
01743 
01744             bug = self.crashdb.launchpad.bugs[id]
01745             tasks = list(bug.bug_tasks)
01746             assert len(tasks) == 1
01747             t = tasks[0]
01748             t.status = 'New'
01749             t.lp_save()
01750 
01751         def _verify_marked_regression(self, id):
01752             '''Verify that report ID is marked as regression.'''
01753 
01754             bug = self.crashdb.launchpad.bugs[id]
01755             self.assertTrue('regression-retracer' in bug.tags)
01756 
01757         def test_project(self):
01758             '''reporting crashes against a project instead of a distro'''
01759 
01760             launchpad_instance = os.environ.get('APPORT_LAUNCHPAD_INSTANCE') or 'staging'
01761             # crash database for langpack-o-matic project (this does not have
01762             # packages in any distro)
01763             crashdb = CrashDatabase(os.environ.get('LP_CREDENTIALS'),
01764                                     {'project': 'langpack-o-matic',
01765                                      'launchpad_instance': launchpad_instance})
01766             self.assertEqual(crashdb.distro, None)
01767 
01768             # create Python crash report
01769             r = apport.Report('Crash')
01770             r['ExecutablePath'] = '/bin/foo'
01771             r['Traceback'] = '''Traceback (most recent call last):
01772   File "/bin/foo", line 67, in fuzz
01773     print(weird)
01774 NameError: global name 'weird' is not defined'''
01775             r.add_os_info()
01776             r.add_user_info()
01777             self.assertEqual(r.standard_title(),
01778                              "foo crashed with NameError in fuzz(): global name 'weird' is not defined")
01779 
01780             # file it
01781             bug_target = self._get_bug_target(crashdb, r)
01782             self.assertEqual(bug_target.name, 'langpack-o-matic')
01783 
01784             id = self._file_bug(bug_target, r)
01785             self.assertTrue(id > 0)
01786             sys.stderr.write('(https://%s/bugs/%i) ' % (self.hostname, id))
01787 
01788             # update
01789             r = crashdb.download(id)
01790             r['StacktraceTop'] = 'read () from /lib/libc.6.so\nfoo (i=1) from /usr/lib/libfoo.so'
01791             r['Stacktrace'] = 'long\ntrace'
01792             r['ThreadStacktrace'] = 'thread\neven longer\ntrace'
01793             crashdb.update_traces(id, r, 'good retrace!')
01794             r = crashdb.download(id)
01795 
01796             # test fixed version
01797             self.assertEqual(crashdb.get_fixed_version(id), None)
01798             crashdb.close_duplicate(r, id, self.get_uncommon_description_report())
01799             self.assertEqual(crashdb.duplicate_of(id), self.get_uncommon_description_report())
01800             self.assertEqual(crashdb.get_fixed_version(id), 'invalid')
01801             crashdb.close_duplicate(r, id, None)
01802             self.assertEqual(crashdb.duplicate_of(id), None)
01803             self.assertEqual(crashdb.get_fixed_version(id), None)
01804 
01805         def test_download_robustness(self):
01806             '''download() of uncommon description formats'''
01807 
01808             # only ProblemType/Architecture/DistroRelease in description
01809             r = self.crashdb.download(self.get_uncommon_description_report())
01810             self.assertEqual(r['ProblemType'], 'Package')
01811             self.assertEqual(r['Architecture'], 'amd64')
01812             self.assertTrue(r['DistroRelease'].startswith('Ubuntu '))
01813 
01814         def test_escalation(self):
01815             '''Escalating bugs with more than 10 duplicates'''
01816 
01817             launchpad_instance = os.environ.get('APPORT_LAUNCHPAD_INSTANCE') or 'staging'
01818             db = CrashDatabase(os.environ.get('LP_CREDENTIALS'),
01819                                {'distro': 'ubuntu',
01820                                 'launchpad_instance': launchpad_instance,
01821                                 'escalation_tag': 'omgkittens',
01822                                 'escalation_subscription': 'apport-hackers'})
01823 
01824             count = 0
01825             p = db.launchpad.people[db.options['escalation_subscription']].self_link
01826             first_dup = 59
01827             try:
01828                 for b in range(first_dup, first_dup + 13):
01829                     count += 1
01830                     sys.stderr.write('%i ' % b)
01831                     db.close_duplicate(apport.Report(), b, self.get_segv_report())
01832                     b = db.launchpad.bugs[self.get_segv_report()]
01833                     has_escalation_tag = db.options['escalation_tag'] in b.tags
01834                     has_escalation_subscription = any([s.person_link == p for s in b.subscriptions])
01835                     if count <= 10:
01836                         self.assertFalse(has_escalation_tag)
01837                         self.assertFalse(has_escalation_subscription)
01838                     else:
01839                         self.assertTrue(has_escalation_tag)
01840                         self.assertTrue(has_escalation_subscription)
01841             finally:
01842                 for b in range(first_dup, first_dup + count):
01843                     sys.stderr.write('R%i ' % b)
01844                     db.close_duplicate(apport.Report(), b, None)
01845             sys.stderr.write('\n')
01846 
01847         def test_marking_python_task_mangle(self):
01848             '''source package task fixup for marking interpreter crashes'''
01849 
01850             self._mark_needs_dupcheck(self.get_python_report())
01851             unchecked_before = self.crashdb.get_dup_unchecked()
01852             self.assertTrue(self.get_python_report() in unchecked_before)
01853 
01854             # add an upstream task, and remove the package name from the
01855             # package task; _mark_dup_checked is supposed to restore the
01856             # package name
01857             b = self.crashdb.launchpad.bugs[self.get_python_report()]
01858             if b.private:
01859                 b.private = False
01860                 b.lp_save()
01861             t = b.bug_tasks[0]
01862             t.target = self.crashdb.launchpad.distributions['ubuntu']
01863             t.lp_save()
01864             b.addTask(target=self.crashdb.launchpad.projects['coreutils'])
01865 
01866             r = self.crashdb.download(self.get_python_report())
01867             self.crashdb._mark_dup_checked(self.get_python_report(), r)
01868 
01869             unchecked_after = self.crashdb.get_dup_unchecked()
01870             self.assertFalse(self.get_python_report() in unchecked_after)
01871             self.assertEqual(unchecked_before,
01872                              unchecked_after.union(set([self.get_python_report()])))
01873 
01874             # upstream task should be unmodified
01875             b = self.crashdb.launchpad.bugs[self.get_python_report()]
01876             self.assertEqual(b.bug_tasks[0].bug_target_name, 'coreutils')
01877             self.assertEqual(b.bug_tasks[0].status, 'New')
01878             self.assertEqual(b.bug_tasks[0].importance, 'Undecided')
01879 
01880             # package-less distro task should have package name fixed
01881             self.assertEqual(b.bug_tasks[1].bug_target_name, 'coreutils (Ubuntu)')
01882             self.assertEqual(b.bug_tasks[1].status, 'New')
01883             self.assertEqual(b.bug_tasks[1].importance, 'Medium')
01884 
01885             # should not confuse get_fixed_version()
01886             self.assertEqual(self.crashdb.get_fixed_version(self.get_python_report()), None)
01887 
01888         @classmethod
01889         def _generate_sigsegv_report(klass, signal='11'):
01890             '''Create a test executable which will die with a SIGSEGV, generate a
01891             core dump for it, create a problem report with those two arguments
01892             (ExecutablePath and CoreDump) and call add_gdb_info().
01893 
01894             Return the apport.report.Report.
01895             '''
01896             workdir = None
01897             orig_cwd = os.getcwd()
01898             pr = apport.report.Report()
01899             try:
01900                 workdir = tempfile.mkdtemp()
01901                 atexit.register(shutil.rmtree, workdir)
01902                 os.chdir(workdir)
01903 
01904                 # create a test executable
01905                 with open('crash.c', 'w') as fd:
01906                     fd.write('''
01907 int f(x) {
01908     int* p = 0; *p = x;
01909     return x+1;
01910 }
01911 int main() { return f(42); }
01912 ''')
01913                 assert subprocess.call(['gcc', '-g', 'crash.c', '-o', 'crash']) == 0
01914                 assert os.path.exists('crash')
01915 
01916                 # call it through gdb and dump core
01917                 subprocess.call(['gdb', '--batch', '--ex', 'run', '--ex',
01918                                  'generate-core-file core', './crash'], stdout=subprocess.PIPE)
01919                 assert os.path.exists('core')
01920                 subprocess.check_call(['sync'])
01921                 assert subprocess.call(['readelf', '-n', 'core'],
01922                                        stdout=subprocess.PIPE) == 0
01923 
01924                 pr['ExecutablePath'] = os.path.join(workdir, 'crash')
01925                 pr['CoreDump'] = (os.path.join(workdir, 'core'),)
01926                 pr['Signal'] = signal
01927 
01928                 pr.add_gdb_info()
01929             finally:
01930                 os.chdir(orig_cwd)
01931 
01932             return pr
01933 
01934     unittest.main()