Skip to content

refactor(retracer): split callback into smaller functions #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

refactor(retracer): split callback into smaller functions #25

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7b1d5c1
refactor(retracer): extract build_report from callback
Copilot Jun 15, 2026
05f3545
refactor(retracer): extract decompress_core from callback
Copilot Jun 15, 2026
00f675b
refactor(retracer): extract gdb_core_check from callback
Copilot Jun 15, 2026
a686bff
refactor(retracer): extract check_retraceable from callback
Copilot Jun 15, 2026
ca543c8
refactor(retracer): extract run_apport_retrace from callback
Copilot Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
230 changes: 129 additions & 101 deletions src/retracer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,58 +364,21 @@ def save_crash(self, report, oops_id, core_file):
with open(failed_crash, "wb") as fp:
report.write(fp)

@prefix_log_with_amqp_message
def callback(self, msg):
self._processing_callback = True
log("Processing.")
self.msg_body = ensure_str(msg.body)
oops_id, provider = self.msg_body.split(":", 1)
try:
col = cassandra_schema.OOPS.get_as_dict(key=oops_id.encode())
except cassandra_schema.DoesNotExist:
# We do not have enough information at this point to be able to
# remove this from the retracing row in the Indexes CF. Throw it
# back on the queue and hope that eventual consistency works its
# magic by then.
log("Unable to find in OOPS CF.")
self.requeue(msg, oops_id)

metrics.meter("could_not_find_oops")
return

# ack the message very early, to prevent them from staying forever in
# the queue in case the retracer gets OOM-killed
log("ack'ing message from queue")
msg.channel.basic_ack(msg.delivery_tag)

# There are some items still in amqp queue that have already been
# retraced, check for this and ack the message.
# N.B.: This only works in some cases because we don't mark a report as
# having been retraced e.g. there is no Retrace column in keys
if "RetraceFailureReason" in list(col.keys()) or "RetraceOutdatedPackages" in list(
col.keys()
):
log("already retraced OOPS.")
# also try to remove the core file
self.remove(oops_id)
return

# Check to see if there is an UnreportableReason so we can log more
# information about failures to retrace.
unreportable_reason = ""
if "UnreportableReason" in list(col.keys()):
unreportable_reason = col["UnreportableReason"]

work_path = self.write_bucket_to_disk(oops_id)

if not work_path or not work_path.exists():
log("Could not find %s" % work_path)
self.failed_to_process(msg, oops_id)
return
def build_report(self, col):
report = Report()

core_file = work_path / "core"
report_path = work_path / "crash"
for k in col:
try:
report[k] = col[k]
except (AssertionError, ValueError):
# apport raises an ValueError if a key is invalid
# e.g. /usr/bin/media-hub-server became a key somehow,
# and this doesn't need to be part of the report used
# for retracing
continue
return report

def decompress_core(self, report_path, core_file):
try:
with open(core_file, "wb") as fp:
log("Decompressing to %s" % core_file)
Expand All @@ -426,49 +389,19 @@ def callback(self, msg):
fp.write(block)
except Exception as e:
log("Failed to decompress core: %s" % str(e))
# We couldn't decompress this, so there's no value in trying again.
self.remove(oops_id)
self.update_time_to_retrace(msg)
# probably incomplete cores from armhf?
metrics.meter("retrace.failed")
metrics.meter("retrace.failed.%s" % self.architecture)
metrics.meter("retrace.failure.decompression")
metrics.meter("retrace.failure.decompression.%s" % self.architecture)
rm_eff(work_path)
return
return False
return True

def gdb_core_check(self, core_file):
# confirm that gdb thinks the core file is good
gdb_cmd = [self.gdb_path, "--batch", "--ex", "target core %s" % core_file]
proc = Popen(gdb_cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, errors="ignore")
(out, err) = proc.communicate()
if "is truncated: expected core file size" in err or "not a core dump" in err:
# Not a core file, there's no value in trying again.
self.remove(oops_id)
self.update_time_to_retrace(msg)
log("Not a core dump per gdb.")
if unreportable_reason:
log("UnreportableReason is: %s" % unreportable_reason)
metrics.meter("retrace.failed")
metrics.meter("retrace.failed.%s" % self.architecture)
metrics.meter("retrace.failure.gdb_core_check")
metrics.meter("retrace.failure.gdb_core_check.%s" % self.architecture)
rm_eff(work_path)
return

report = Report()

for k in col:
try:
report[k] = col[k]
except (AssertionError, ValueError):
# apport raises an ValueError if a key is invalid
# e.g. /usr/bin/media-hub-server became a key somehow,
# and this doesn't need to be part of the report used
# for retracing
continue
return False
return True

# these will not change after retracing
architecture = report.get("Architecture", "")
def check_retraceable(self, report, unreportable_reason):
release = report.get("DistroRelease", "")
bad = "[^-a-zA-Z0-9_.() ]+"
retraceable = utils.retraceable_release(release)
Expand All @@ -485,27 +418,15 @@ def callback(self, msg):
log("UnreportableReason is: %s" % unreportable_reason)
metrics.meter("retrace.failed.foreign")
retraceable = False
# srcpackage = report.get('SourcePackage', '')
# if srcpackage in ['kodi', 'mysql-workbench'] and release == 'Ubuntu 18.04':
# 2018-06-13 gdb is hanging trying to retrace these so put them at
# the end of the line.
# log('Requeueing an Ubuntu 18.04 %s crash.' % srcpackage)
# self.requeue(msg, oops_id)
# rm_eff(path)
# return

invalid = re.search(bad, release) or len(release) > 1024
if invalid:
metrics.meter("retrace.failed.invalid")
if not release or invalid or not retraceable:
self.remove(oops_id)
self.update_time_to_retrace(msg)
rm_eff(work_path)
return

with open(report_path, "wb") as fp:
report.write(fp)
return False
return True

def run_apport_retrace(self, report_path, core_file, release, architecture):
try:
retrace_msg = "Retracing {}".format(self.msg_body)
sandbox, cache = self.setup_cache(self.sandbox_dir, release)
Expand Down Expand Up @@ -578,6 +499,113 @@ def callback(self, msg):
log("Removing %s" % cache)
shutil.rmtree(cache)
os.mkdir(cache)
return proc, out, err, day_key, retracing_start_time

@prefix_log_with_amqp_message
def callback(self, msg):
self._processing_callback = True
log("Processing.")
self.msg_body = ensure_str(msg.body)
oops_id, provider = self.msg_body.split(":", 1)
try:
col = cassandra_schema.OOPS.get_as_dict(key=oops_id.encode())
except cassandra_schema.DoesNotExist:
# We do not have enough information at this point to be able to
# remove this from the retracing row in the Indexes CF. Throw it
# back on the queue and hope that eventual consistency works its
# magic by then.
log("Unable to find in OOPS CF.")
self.requeue(msg, oops_id)

metrics.meter("could_not_find_oops")
return

# ack the message very early, to prevent them from staying forever in
# the queue in case the retracer gets OOM-killed
log("ack'ing message from queue")
msg.channel.basic_ack(msg.delivery_tag)

# There are some items still in amqp queue that have already been
# retraced, check for this and ack the message.
# N.B.: This only works in some cases because we don't mark a report as
# having been retraced e.g. there is no Retrace column in keys
if "RetraceFailureReason" in list(col.keys()) or "RetraceOutdatedPackages" in list(
col.keys()
):
log("already retraced OOPS.")
# also try to remove the core file
self.remove(oops_id)
return

# Check to see if there is an UnreportableReason so we can log more
# information about failures to retrace.
unreportable_reason = ""
if "UnreportableReason" in list(col.keys()):
unreportable_reason = col["UnreportableReason"]

work_path = self.write_bucket_to_disk(oops_id)

if not work_path or not work_path.exists():
log("Could not find %s" % work_path)
self.failed_to_process(msg, oops_id)
return

core_file = work_path / "core"
report_path = work_path / "crash"

if not self.decompress_core(report_path, core_file):
# We couldn't decompress this, so there's no value in trying again.
self.remove(oops_id)
self.update_time_to_retrace(msg)
# probably incomplete cores from armhf?
metrics.meter("retrace.failed")
metrics.meter("retrace.failed.%s" % self.architecture)
metrics.meter("retrace.failure.decompression")
metrics.meter("retrace.failure.decompression.%s" % self.architecture)
rm_eff(work_path)
return

# confirm that gdb thinks the core file is good
if not self.gdb_core_check(core_file):
# Not a core file, there's no value in trying again.
self.remove(oops_id)
self.update_time_to_retrace(msg)
log("Not a core dump per gdb.")
if unreportable_reason:
log("UnreportableReason is: %s" % unreportable_reason)
metrics.meter("retrace.failed")
metrics.meter("retrace.failed.%s" % self.architecture)
metrics.meter("retrace.failure.gdb_core_check")
metrics.meter("retrace.failure.gdb_core_check.%s" % self.architecture)
rm_eff(work_path)
return

report = self.build_report(col)

# these will not change after retracing
architecture = report.get("Architecture", "")
release = report.get("DistroRelease", "")
# srcpackage = report.get('SourcePackage', '')
# if srcpackage in ['kodi', 'mysql-workbench'] and release == 'Ubuntu 18.04':
# 2018-06-13 gdb is hanging trying to retrace these so put them at
# the end of the line.
# log('Requeueing an Ubuntu 18.04 %s crash.' % srcpackage)
# self.requeue(msg, oops_id)
# rm_eff(path)
# return

if not self.check_retraceable(report, unreportable_reason):
self.remove(oops_id)
self.update_time_to_retrace(msg)
rm_eff(work_path)
return

with open(report_path, "wb") as fp:
report.write(fp)

proc, out, err, day_key, retracing_start_time = self.run_apport_retrace(
report_path, core_file, release, architecture
)

try:
if proc.returncode != 0:
Expand Down