Skip to content

Commit 117ff5a

Browse files
gh-159207: Store Tachyon samples in bounded columnar blocks (GH-160313)
Store samples in independently decodable columnar blocks so binary replay can process a profile without decompressing the complete sample stream. Use per-stream timestamp and stack deltas, status bitmaps, and compact frame IDs within each block.
1 parent 0d5e2ed commit 117ff5a

12 files changed

Lines changed: 2724 additions & 1129 deletions

File tree

‎InternalDocs/profiling_binary_format.md‎

Lines changed: 198 additions & 98 deletions
Large diffs are not rendered by default.

‎Lib/profiling/sampling/binary_collector.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ def get_stats(self):
146146
"""Get encoding statistics.
147147
148148
Returns:
149-
Dict with encoding statistics including repeat/full/suffix/pop-push
149+
Dict with encoding statistics including repeat/full/suffix/reuse
150150
record counts, frames written/saved, and compression ratio.
151151
"""
152152
return self._writer.get_stats()

‎Lib/profiling/sampling/sample.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -411,22 +411,22 @@ def _print_binary_stats(self, collector):
411411
repeat_samples = stats.get('repeat_samples', 0)
412412
full_records = stats.get('full_records', 0)
413413
suffix_records = stats.get('suffix_records', 0)
414-
pop_push_records = stats.get('pop_push_records', 0)
414+
reuse_records = stats.get('reuse_records', 0)
415415
total_records = stats.get('total_records', 0)
416416

417417
if total_records > 0:
418418
repeat_pct = repeat_records / total_records * 100
419419
full_pct = full_records / total_records * 100
420420
suffix_pct = suffix_records / total_records * 100
421-
pop_push_pct = pop_push_records / total_records * 100
421+
reuse_pct = reuse_records / total_records * 100
422422
else:
423-
repeat_pct = full_pct = suffix_pct = pop_push_pct = 0
423+
repeat_pct = full_pct = suffix_pct = reuse_pct = 0
424424

425425
print(f" Records: {total_records:,}")
426426
print(f" RLE repeat: {repeat_records:,} ({ANSIColors.GREEN}{repeat_pct:.1f}%{ANSIColors.RESET}) [{repeat_samples:,} samples]")
427427
print(f" Full stack: {full_records:,} ({full_pct:.1f}%)")
428428
print(f" Suffix match: {suffix_records:,} ({suffix_pct:.1f}%)")
429-
print(f" Pop-push: {pop_push_records:,} ({pop_push_pct:.1f}%)")
429+
print(f" Stack reuse: {reuse_records:,} ({reuse_pct:.1f}%)")
430430

431431
frames_written = stats.get('total_frames_written', 0)
432432
frames_saved = stats.get('frames_saved', 0)

‎Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py‎

Lines changed: 773 additions & 39 deletions
Large diffs are not rendered by default.

‎Lib/test/test_profiling/test_sampling_profiler/test_cli.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -954,15 +954,15 @@ def test_cli_replay_reader_errors_exit_cleanly(self):
954954
mock.patch("sys.argv", ["profiling.sampling.cli", "replay", profile]),
955955
mock.patch(
956956
"profiling.sampling.cli.BinaryReader",
957-
side_effect=ValueError("Unsupported format version 2"),
957+
side_effect=ValueError("Unsupported format version 3"),
958958
),
959959
):
960960
with self.assertRaises(SystemExit) as cm:
961961
main()
962962

963963
self.assertEqual(
964964
str(cm.exception),
965-
"Error: Unsupported format version 2",
965+
"Error: Unsupported format version 3",
966966
)
967967

968968
def test_cli_replay_propagates_recorded_mode(self):

‎Makefile.pre.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3526,7 +3526,7 @@ MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@
35263526
MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@
35273527
MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_openssl_mem.h
35283528
MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
3529-
MODULE__REMOTE_DEBUGGING_DEPS=$(srcdir)/Modules/_remote_debugging/_remote_debugging.h $(srcdir)/Modules/_remote_debugging/gc_stats.h
3529+
MODULE__REMOTE_DEBUGGING_DEPS=$(srcdir)/Modules/_remote_debugging/_remote_debugging.h $(srcdir)/Modules/_remote_debugging/binary_io.h $(srcdir)/Modules/_remote_debugging/gc_stats.h
35303530

35313531
# HACL*-based cryptographic primitives
35323532
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The :mod:`profiling.sampling` binary output format now stores samples in
2+
independent blocks. This is a breaking format change: Python 3.16 does not
3+
support reading profiles written in the previous format.

0 commit comments

Comments
 (0)