Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 61 additions & 10 deletions pythonbuild/mirror.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,19 @@

import argparse
import concurrent.futures
import hashlib
import os
import re
from contextlib import closing
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Sequence

import boto3
from boto3.exceptions import S3UploadFailedError
from boto3.s3.transfer import TransferConfig
from botocore.config import Config
from botocore.exceptions import ClientError
from mypy_boto3_s3.client import S3Client as Boto3S3Client

CACHE_CONTROL = "public, max-age=31536000, immutable"
Expand All @@ -48,6 +52,7 @@ class MirrorError(RuntimeError):
class UploadEntry:
source_name: str
dest_name: str
sha256: str


def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
Expand Down Expand Up @@ -96,24 +101,24 @@ def infer_build_datetime(dist_dir: Path) -> str:
return datetimes.pop()


def parse_shasums(shasums_path: Path) -> list[str]:
def parse_shasums(shasums_path: Path) -> list[tuple[str, str]]:
if not shasums_path.exists():
raise SystemExit(f"{shasums_path} not found")

filenames: list[str] = []
checksums: list[tuple[str, str]] = []
for line in shasums_path.read_text().splitlines():
line = line.strip()
if not line:
continue

try:
_digest, filename = line.split(maxsplit=1)
digest, filename = line.split(maxsplit=1)
except ValueError as e:
raise SystemExit(f"malformed line in {shasums_path}: {line!r}") from e

filenames.append(filename.lstrip("*"))
checksums.append((filename.lstrip("*"), digest.lower()))

return filenames
return checksums


def destination_to_source_name(dest_name: str, tag: str, build_datetime: str) -> str:
Expand All @@ -140,17 +145,19 @@ def build_upload_entries(
dist_dir: Path, tag: str
) -> tuple[list[UploadEntry], list[str]]:
build_datetime = infer_build_datetime(dist_dir)
dest_names = parse_shasums(dist_dir / "SHA256SUMS")
checksums = parse_shasums(dist_dir / "SHA256SUMS")

uploads: list[UploadEntry] = []
missing: list[str] = []

for dest_name in dest_names:
for dest_name, sha256 in checksums:
source_name = destination_to_source_name(dest_name, tag, build_datetime)
if not (dist_dir / source_name).exists():
missing.append(source_name)
continue
uploads.append(UploadEntry(source_name=source_name, dest_name=dest_name))
uploads.append(
UploadEntry(source_name=source_name, dest_name=dest_name, sha256=sha256)
)

return uploads, missing

Expand All @@ -166,6 +173,8 @@ def make_s3_client() -> tuple[Boto3S3Client, TransferConfig]:
signature_version="s3v4",
retries={"max_attempts": MAX_ATTEMPTS, "mode": "standard"},
s3={"addressing_style": "path"},
# The lock recovery check uses the manifest's full-object SHA-256.
response_checksum_validation="when_required",
)
}
if endpoint_url:
Expand All @@ -174,7 +183,7 @@ def make_s3_client() -> tuple[Boto3S3Client, TransferConfig]:
client_kwargs["region_name"] = region_name

session = boto3.session.Session()
client = session.client("s3", **client_kwargs)
client: Boto3S3Client = session.client("s3", **client_kwargs)
transfer_config = TransferConfig(
multipart_threshold=SINGLE_UPLOAD_THRESHOLD,
)
Expand All @@ -189,7 +198,9 @@ class S3MirrorClient:
dry_run: bool
existing_keys: set[str]

def upload_file(self, bucket: str, key: str, path: Path) -> None:
def upload_file(
self, bucket: str, key: str, path: Path, sha256: str | None = None
) -> None:
if key in self.existing_keys:
print(f"skipping existing s3://{bucket}/{key}")
return
Expand All @@ -210,8 +221,47 @@ def upload_file(self, bucket: str, key: str, path: Path) -> None:
Config=self.transfer_config,
)
except Exception as e:
# boto3 wraps the original ClientError in upload_file().
error = e
if isinstance(e, S3UploadFailedError):
error = e.__cause__ or e.__context__
if (
isinstance(error, ClientError)
and error.operation_name in ("PutObject", "CompleteMultipartUpload")
and error.response.get("Error", {}).get("Code")
== "ObjectLockedByBucketPolicy"
):
self._verify_locked_object(self.client, bucket, key, path, sha256)
return
raise MirrorError(f"failed to upload {path.name}: {e}") from e

@staticmethod
def _verify_locked_object(
client: Boto3S3Client, bucket: str, key: str, path: Path, sha256: str | None
) -> None:
try:
# SHA256SUMS has no entry for itself.
if sha256 is None:
with path.open("rb") as local:
sha256 = hashlib.file_digest(local, "sha256").hexdigest()

response = client.get_object(Bucket=bucket, Key=key)
digest = hashlib.sha256()
with closing(response["Body"]) as body:
for chunk in body.iter_chunks(1024 * 1024):
digest.update(chunk)
except Exception as e:
raise MirrorError(
f"failed to verify locked object s3://{bucket}/{key}: {e}"
) from e

actual = digest.hexdigest()
if actual != sha256:
raise MirrorError(
f"SHA-256 mismatch for s3://{bucket}/{key}: expected {sha256}, got {actual}"
)
print(f"verified existing s3://{bucket}/{key}")


def main(argv: Sequence[str] | None = None) -> int:
args = parse_args(argv)
Expand Down Expand Up @@ -257,6 +307,7 @@ def main(argv: Sequence[str] | None = None) -> int:
args.bucket,
f"{args.prefix}{entry.dest_name}",
args.dist / entry.source_name,
entry.sha256,
)
for entry in uploads
]
Expand Down
166 changes: 156 additions & 10 deletions tests/test_mirror.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,20 @@

from __future__ import annotations

import io
import os
import tempfile
import unittest
from hashlib import sha256
from pathlib import Path
from unittest import mock

from boto3.s3.transfer import TransferConfig
from botocore.response import StreamingBody
from botocore.stub import ANY, Stubber

from pythonbuild.mirror import (
CACHE_CONTROL,
MirrorError,
S3MirrorClient,
UploadEntry,
Expand All @@ -21,6 +27,11 @@
make_s3_client,
)

FULL_SOURCE = "cpython-3.12.10-x86_64-unknown-linux-gnu-pgo+lto-20260324T1200.tar.zst"
FULL_DEST = "cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst"
FULL_SHA256 = sha256(b"full").hexdigest()
INSTALL_SHA256 = sha256(b"install").hexdigest()


class MirrorTests(unittest.TestCase):
def test_make_s3_client_uses_single_uploads_below_threshold(self) -> None:
Expand Down Expand Up @@ -63,17 +74,14 @@ def test_destination_to_source_name_rejects_wrong_tag(self) -> None:
def test_build_upload_entries(self) -> None:
with tempfile.TemporaryDirectory() as td:
dist = Path(td)
(
dist
/ "cpython-3.12.10-x86_64-unknown-linux-gnu-pgo+lto-20260324T1200.tar.zst"
).write_bytes(b"full")
(dist / FULL_SOURCE).write_bytes(b"full")
(
dist
/ "cpython-3.12.10-x86_64-unknown-linux-gnu-install_only-20260324T1200.tar.gz"
).write_bytes(b"install")
(dist / "SHA256SUMS").write_text(
"abc cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst\n"
"def cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-install_only.tar.gz\n"
f"{FULL_SHA256} {FULL_DEST}\n"
f"{INSTALL_SHA256} *cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-install_only.tar.gz\n"
)

uploads, missing = build_upload_entries(dist, "20260324")
Expand All @@ -83,12 +91,14 @@ def test_build_upload_entries(self) -> None:
uploads,
[
UploadEntry(
source_name="cpython-3.12.10-x86_64-unknown-linux-gnu-pgo+lto-20260324T1200.tar.zst",
dest_name="cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst",
source_name=FULL_SOURCE,
dest_name=FULL_DEST,
sha256=FULL_SHA256,
),
UploadEntry(
source_name="cpython-3.12.10-x86_64-unknown-linux-gnu-install_only-20260324T1200.tar.gz",
dest_name="cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-install_only.tar.gz",
sha256=INSTALL_SHA256,
),
],
)
Expand Down Expand Up @@ -133,7 +143,7 @@ def test_main_reports_all_upload_errors(self) -> None:
"def cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-install_only.tar.gz\n"
)

def fail_upload(bucket: str, key: str, path: Path) -> None:
def fail_upload(bucket: str, key: str, path: Path, sha256: str) -> None:
raise MirrorError(f"failed for {path.name}")

client = mock.Mock()
Expand All @@ -147,7 +157,7 @@ def fail_upload(bucket: str, key: str, path: Path) -> None:
S3MirrorClient,
"upload_file",
side_effect=fail_upload,
),
) as upload_file,
):
with self.assertRaises(SystemExit) as cm:
main(
Expand All @@ -169,6 +179,142 @@ def fail_upload(bucket: str, key: str, path: Path) -> None:
f"- failed for {install_only_path.name}\n"
f"- failed for {full_path.name}",
)
self.assertEqual(upload_file.call_count, 2)


class MirrorUploadTests(unittest.TestCase):
def setUp(self) -> None:
self.dist = Path(self.enterContext(tempfile.TemporaryDirectory()))
self.path = self.dist / FULL_SOURCE
self.path.write_bytes(b"full")
self.bucket = "bucket"
self.prefix = "prefix/"
self.key = self.prefix + FULL_DEST
self.object_params = {"Bucket": self.bucket, "Key": self.key}
self.put_params = {
**self.object_params,
"Body": ANY,
"CacheControl": CACHE_CONTROL,
}
with mock.patch.dict(
os.environ,
{
"AWS_ACCESS_KEY_ID": "testing",
"AWS_SECRET_ACCESS_KEY": "testing",
"AWS_DEFAULT_REGION": "us-east-1",
"AWS_ENDPOINT_URL": "http://127.0.0.1:1",
"AWS_REQUEST_CHECKSUM_CALCULATION": "when_required",
},
clear=True,
):
self.client, _ = make_s3_client()
self.addCleanup(self.client.close)
self.stubber = self.enterContext(Stubber(self.client))
self.addCleanup(self.stubber.assert_no_pending_responses)
self.transfer_config = TransferConfig(preferred_transfer_client="classic")
self.mirror = S3MirrorClient(
client=self.client,
transfer_config=self.transfer_config,
dry_run=False,
existing_keys=set(),
)

def stub_locked_put(self, key: str) -> None:
self.stubber.add_client_error(
"put_object",
"ObjectLockedByBucketPolicy",
http_status_code=409,
expected_params={**self.put_params, "Key": key},
)

def stub_get_object(self, key: str, data: bytes) -> io.BytesIO:
stream = io.BytesIO(data)
self.stubber.add_response(
"get_object",
{"Body": StreamingBody(stream, len(data))},
{"Bucket": self.bucket, "Key": key},
)
return stream

def test_recovers_locked_archive_and_manifest_after_listing_miss(self) -> None:
manifest = f"{FULL_SHA256} {FULL_DEST}\n".encode()
(self.dist / "SHA256SUMS").write_bytes(manifest)
# The release manifest is the source of truth for archive recovery.
self.path.write_bytes(b"different staged bytes")
self.stubber.add_response(
"list_objects_v2", {}, {"Bucket": self.bucket, "Prefix": self.prefix}
)
streams = []
for key, data in ((self.key, b"full"), (self.prefix + "SHA256SUMS", manifest)):
self.stub_locked_put(key)
streams.append(self.stub_get_object(key, data))

with mock.patch(
"pythonbuild.mirror.make_s3_client",
return_value=(self.client, self.transfer_config),
):
self.assertEqual(
main(
[
"--dist",
str(self.dist),
"--tag",
"20260324",
"--bucket",
self.bucket,
"--prefix",
self.prefix,
]
),
0,
)
self.assertTrue(all(stream.closed for stream in streams))

def test_locked_object_with_mismatched_bytes_fails(self) -> None:
self.stub_locked_put(self.key)
self.stub_get_object(self.key, b"wrong")

with self.assertRaises(MirrorError) as cm:
self.mirror.upload_file(self.bucket, self.key, self.path, FULL_SHA256)
for detail in (self.key, FULL_SHA256, sha256(b"wrong").hexdigest()):
self.assertIn(detail, str(cm.exception))

def test_unrelated_conflict_is_not_reconciled(self) -> None:
self.stubber.add_client_error(
"put_object",
"OperationAborted",
http_status_code=409,
expected_params=self.put_params,
)

with self.assertRaisesRegex(MirrorError, "OperationAborted"):
self.mirror.upload_file(self.bucket, self.key, self.path, FULL_SHA256)

def test_lock_recovery_fails_when_exact_key_is_missing(self) -> None:
self.stub_locked_put(self.key)
self.stubber.add_client_error(
"get_object",
"NoSuchKey",
http_status_code=404,
expected_params=self.object_params,
)
with self.assertRaisesRegex(MirrorError, "NoSuchKey"):
self.mirror.upload_file(self.bucket, self.key, self.path, FULL_SHA256)

def test_locked_multipart_completion_aborts_then_verifies(self) -> None:
self.transfer_config.multipart_threshold = 1
self.stubber.add_response("create_multipart_upload", {"UploadId": "upload-id"})
upload_params = {**self.object_params, "UploadId": "upload-id"}
self.stubber.add_response("upload_part", {"ETag": "part-etag"})
self.stubber.add_client_error(
"complete_multipart_upload",
"ObjectLockedByBucketPolicy",
http_status_code=409,
)
self.stubber.add_response("abort_multipart_upload", {}, upload_params)
self.stub_get_object(self.key, b"full")

self.mirror.upload_file(self.bucket, self.key, self.path, FULL_SHA256)


if __name__ == "__main__":
Expand Down
Loading