From 5cf804de53c4aae681193a4e7d90f4a159e7cb20 Mon Sep 17 00:00:00 2001 From: Snigdha Date: Fri, 4 Sep 2026 16:29:17 -0400 Subject: [PATCH] Compare protocol-test list headers semantically; add runner skip list --- ...gfix-protocol-tests-header-comparison.json | 6 + .../aws/testing/AwsProtocolTestHelpers.h | 17 +- tools/scripts/run_protocol_tests.py | 154 +++++++++++++++++- 3 files changed, 172 insertions(+), 5 deletions(-) create mode 100644 .changelog/bugfix-protocol-tests-header-comparison.json diff --git a/.changelog/bugfix-protocol-tests-header-comparison.json b/.changelog/bugfix-protocol-tests-header-comparison.json new file mode 100644 index 000000000000..5bdc22b120c6 --- /dev/null +++ b/.changelog/bugfix-protocol-tests-header-comparison.json @@ -0,0 +1,6 @@ +{ + "type": "bugfix", + "category": "protocol-tests", + "contributor": "pulimsr", + "description": "Compare list-valued HTTP headers semantically in the generated protocol tests: whitespace after the comma (RFC 7230 OWS) is normalized so \"a, b, c\" and \"a,b,c\" match, while element order, content, and quoting stay significant. Also add a known-failures skip list to the protocol test runner so meaningful failures are not masked by known-open issues." +} diff --git a/tests/testing-resources/include/aws/testing/AwsProtocolTestHelpers.h b/tests/testing-resources/include/aws/testing/AwsProtocolTestHelpers.h index f22e2b60b78d..f9b7f60a2f12 100644 --- a/tests/testing-resources/include/aws/testing/AwsProtocolTestHelpers.h +++ b/tests/testing-resources/include/aws/testing/AwsProtocolTestHelpers.h @@ -227,6 +227,18 @@ class AwsProtocolTestSuite : public Aws::Testing::AwsCppSdkGTestSuite { EXPECT_STREQ(sortedQuery(expected, expQ).c_str(), sortedQuery(received, recQ).c_str()); } + static Aws::String CanonicalizeHeaderValue(const Aws::String& value) { + const Aws::Vector parts = Aws::Utils::StringUtils::Split(value, ','); + Aws::String joined; + for (const auto& part : parts) { + if (!joined.empty()) { + joined += ","; + } + joined += Aws::Utils::StringUtils::Trim(part.c_str()); + } + return joined; + } + void ValidateRequestSent(const std::function& bodyCompare = ValidateBody) const { ValidateRequestSent(ExpectedRequest(), bodyCompare); } @@ -260,8 +272,9 @@ class AwsProtocolTestSuite : public Aws::Testing::AwsCppSdkGTestSuite { return Aws::Utils::StringUtils::ToLower(hdr.GetKey().c_str()) == Aws::Utils::StringUtils::ToLower(expectedHeader.first.c_str()); }); ASSERT_TRUE(foundIt != receivedHeaders.end()); - ASSERT_STREQ(Aws::Utils::StringUtils::ToLower(expectedHeader.second.c_str()).c_str(), - Aws::Utils::StringUtils::ToLower(foundIt->GetVal().c_str()).c_str()); + // Compare list-valued headers with OWS around commas normalized (RFC 7230). + ASSERT_STREQ(Aws::Utils::StringUtils::ToLower(CanonicalizeHeaderValue(expectedHeader.second).c_str()).c_str(), + Aws::Utils::StringUtils::ToLower(CanonicalizeHeaderValue(foundIt->GetVal()).c_str()).c_str()); } } if (!expected.forbidHeaders.empty()) { diff --git a/tools/scripts/run_protocol_tests.py b/tools/scripts/run_protocol_tests.py index f118a69234a5..d7045fb69a14 100644 --- a/tools/scripts/run_protocol_tests.py +++ b/tools/scripts/run_protocol_tests.py @@ -18,6 +18,148 @@ SCRIPTS_DIR = os.path.dirname(__file__) PROTO_TEST_MOCK_HANDLER = f"{SCRIPTS_DIR}/protocol_tests_mock_server/protocol_tests_mock_server.py" +KNOWN_FAILURES = { + ("rest-xml", "input"): [ + "AllQueryStringTypes.AllQueryStringTypes", + "AllQueryStringTypes.RestXmlSupportsNaNFloatQueryValues", + "AllQueryStringTypes.RestXmlSupportsInfinityFloatQueryValues", + "AllQueryStringTypes.RestXmlSupportsNegativeInfinityFloatQueryValues", + "AllQueryStringTypes.RestXmlZeroAndFalseQueryValues", + "BodyWithXmlName.BodyWithXmlName", + "ConstantQueryString.ConstantQueryString", + "EndpointOperation.RestXmlEndpointTrait", + "EndpointWithHostLabelHeaderOperation.RestXmlEndpointTraitWithHostLabelAndHttpBinding", + "EndpointWithHostLabelOperation.RestXmlEndpointTraitWithHostLabel", + "FlattenedXmlMap.FlattenedXmlMap", + "FlattenedXmlMapWithXmlName.FlattenedXmlMapWithXmlName", + "HttpEnumPayload.RestXmlEnumPayloadRequest", + "HttpPayloadWithUnion.RestXmlHttpPayloadWithUnion", + "HttpPayloadWithXmlName.HttpPayloadWithXmlName", + "HttpPayloadWithXmlNamespaceAndPrefix.HttpPayloadWithXmlNamespaceAndPrefix", + "HttpPayloadWithXmlNamespace.HttpPayloadWithXmlNamespace", + "HttpRequestWithFloatLabels.RestXmlSupportsNaNFloatLabels", + "HttpRequestWithFloatLabels.RestXmlSupportsInfinityFloatLabels", + "HttpRequestWithFloatLabels.RestXmlSupportsNegativeInfinityFloatLabels", + "HttpRequestWithLabelsAndTimestampFormat.HttpRequestWithLabelsAndTimestampFormat", + "HttpRequestWithLabels.InputWithHeadersAndAllParams", + "HttpRequestWithLabels.HttpRequestLabelEscaping", + "HttpStringPayload.RestXmlStringPayloadRequest", + "InputAndOutputWithHeaders.InputAndOutputWithBooleanHeaders", + "InputAndOutputWithHeaders.RestXmlSupportsInfinityFloatHeaderInputs", + "InputAndOutputWithHeaders.RestXmlSupportsNegativeInfinityFloatHeaderInputs", + "NestedXmlMapWithXmlName.NestedXmlMapWithXmlNameSerializes", + "NestedXmlMaps.FlatNestedXmlMapRequest", + "QueryIdempotencyTokenAutoFill.QueryIdempotencyTokenAutoFill", + "QueryPrecedence.RestXmlQueryPrecedence", + "SimpleScalarProperties.RestXmlSupportsNaNFloatInputs", + "SimpleScalarProperties.RestXmlSupportsInfinityFloatInputs", + "SimpleScalarProperties.RestXmlSupportsNegativeInfinityFloatInputs", + "XmlAttributesOnPayload.XmlAttributesOnPayload", + "XmlEmptyMaps.XmlEmptyMaps", + "XmlMapWithXmlNamespace.RestXmlXmlMapWithXmlNamespace", + "XmlNamespaces.XmlNamespaces", + ], + ("json", "input"): [ + "EndpointOperation.AwsJson11EndpointTrait", + "EndpointWithHostLabelOperation.AwsJson11EndpointTraitWithHostLabel", + "HostWithPathOperation.AwsJson11HostWithPath", + "SimpleScalarProperties.AwsJson11SupportsNaNFloatInputs", + "SimpleScalarProperties.AwsJson11SupportsInfinityFloatInputs", + "SimpleScalarProperties.AwsJson11SupportsNegativeInfinityFloatInputs", + ], + ("json_1_0", "input"): [ + "EndpointOperation.AwsJson10EndpointTrait", + "EndpointWithHostLabelOperation.AwsJson10EndpointTraitWithHostLabel", + "HostWithPathOperation.AwsJson10HostWithPath", + "SimpleScalarProperties.AwsJson10SupportsNaNFloatInputs", + "SimpleScalarProperties.AwsJson10SupportsInfinityFloatInputs", + "SimpleScalarProperties.AwsJson10SupportsNegativeInfinityFloatInputs", + ], + ("rest-json", "input"): [ + "HttpEnumPayload.RestJsonEnumPayloadRequest", + "AllQueryStringTypes.RestJsonAllQueryStringTypes", + "AllQueryStringTypes.RestJsonQueryStringEscaping", + "AllQueryStringTypes.RestJsonSupportsNaNFloatQueryValues", + "AllQueryStringTypes.RestJsonSupportsInfinityFloatQueryValues", + "AllQueryStringTypes.RestJsonSupportsNegativeInfinityFloatQueryValues", + "AllQueryStringTypes.RestJsonZeroAndFalseQueryValues", + "ConstantQueryString.RestJsonConstantQueryString", + "EndpointOperation.RestJsonEndpointTrait", + "EndpointWithHostLabelOperation.RestJsonEndpointTraitWithHostLabel", + "HostWithPathOperation.RestJsonHostWithPath", + "HttpChecksumRequired.RestJsonHttpChecksumRequired", + "HttpEmptyPrefixHeaders.RestJsonHttpEmptyPrefixHeadersRequestClient", + "HttpPayloadTraits.RestJsonHttpPayloadTraitsWithBlob", + "HttpRequestWithFloatLabels.RestJsonSupportsNaNFloatLabels", + "HttpRequestWithFloatLabels.RestJsonSupportsInfinityFloatLabels", + "HttpRequestWithFloatLabels.RestJsonSupportsNegativeInfinityFloatLabels", + "HttpRequestWithGreedyLabelInPath.RestJsonHttpRequestWithGreedyLabelInPath", + "HttpRequestWithLabelsAndTimestampFormat.RestJsonHttpRequestWithLabelsAndTimestampFormat", + "HttpRequestWithLabels.RestJsonInputWithHeadersAndAllParams", + "HttpRequestWithLabels.RestJsonHttpRequestLabelEscaping", + "HttpStringPayload.RestJsonStringPayloadRequest", + "InputAndOutputWithHeaders.RestJsonInputAndOutputWithQuotedStringHeaders", + "InputAndOutputWithHeaders.RestJsonInputAndOutputWithBooleanHeaders", + "InputAndOutputWithHeaders.RestJsonSupportsInfinityFloatHeaderInputs", + "InputAndOutputWithHeaders.RestJsonSupportsNegativeInfinityFloatHeaderInputs", + "MediaTypeHeader.MediaTypeHeaderInputBase64", + "QueryIdempotencyTokenAutoFill.RestJsonQueryIdempotencyTokenAutoFill", + "QueryPrecedence.RestJsonQueryPrecedence", + "SimpleScalarProperties.RestJsonSupportsNaNFloatInputs", + "SimpleScalarProperties.RestJsonSupportsInfinityFloatInputs", + "SimpleScalarProperties.RestJsonSupportsNegativeInfinityFloatInputs", + "TestGetNoInputNoPayload.RestJsonHttpGetWithNoInput", + "TestGetNoPayload.RestJsonHttpGetWithNoModeledBody", + "TestGetNoPayload.RestJsonHttpGetWithHeaderMemberNoModeledBody", + "TestPostNoInputNoPayload.RestJsonHttpPostWithNoInput", + "TestPostNoPayload.RestJsonHttpPostWithNoModeledBody", + "TestPostNoPayload.RestJsonHttpWithPostHeaderMemberNoModeledBody", + ], + ("query", "input"): [ + "EndpointOperation.AwsQueryEndpointTrait", + "EndpointWithHostLabelOperation.AwsQueryEndpointTraitWithHostLabel", + "HostWithPathOperation.QueryHostWithPath", + "QueryIdempotencyTokenAutoFill.QueryProtocolIdempotencyTokenAutoFill", + "QueryLists.FlattenedQueryLists", + "QueryMaps.QuerySimpleQueryMapsWithXmlName", + "QueryMaps.QueryFlattenedQueryMaps", + "QueryMaps.QueryQueryMapOfLists", + "SimpleInputParams.AwsQuerySupportsNaNFloatInputs", + "SimpleInputParams.AwsQuerySupportsInfinityFloatInputs", + "SimpleInputParams.AwsQuerySupportsNegativeInfinityFloatInputs", + ], + ("ec2", "input"): [ + "EndpointOperation.Ec2QueryEndpointTrait", + "EndpointWithHostLabelOperation.Ec2QueryEndpointTraitWithHostLabel", + "HostWithPathOperation.Ec2QueryHostWithPath", + "QueryIdempotencyTokenAutoFill.Ec2ProtocolIdempotencyTokenAutoFill", + "QueryIdempotencyTokenAutoFill.Ec2ProtocolIdempotencyTokenAutoFillIsSet", + "QueryLists.Ec2Lists", + "QueryTimestamps.Ec2TimestampsInput", + "SimpleInputParams.Ec2QuerySupportsNaNFloatInputs", + "SimpleInputParams.Ec2QuerySupportsInfinityFloatInputs", + "SimpleInputParams.Ec2QuerySupportsNegativeInfinityFloatInputs", + ], + ("rest-xml", "output"): [ + "HttpEnumPayload.RestXmlEnumPayloadResponse", + ], + ("rest-json", "output"): [ + "DocumentTypeAsMapValue.DocumentTypeAsMapValueOutput", + "HttpEnumPayload.RestJsonEnumPayloadResponse", + ], +} + + +def gtest_filter_for(protocol: str, group: str): + """Build a negative --gtest_filter for the known failures of a binary. + + :return: the filter arg string, or None if nothing is skipped for this binary + """ + skips = KNOWN_FAILURES.get((protocol, group)) + if not skips: + return None + return "--gtest_filter=-" + ":".join(skips) + def parse_arguments() -> dict: """Parse arguments and/or print help @@ -99,10 +241,16 @@ def run(self): if not tests or not len(tests): print(f"No protocol tests found in {self.build_dir}") self.fail = True - for test in tests: + for test, protocol, group in tests: + cmd = [test] + gtest_filter = gtest_filter_for(protocol, group) + if gtest_filter: + cmd.append(gtest_filter) + print(f"Skipping {len(KNOWN_FAILURES[(protocol, group)])} known-failing " + f"test(s) in {protocol}-{group}") with MockHttpServerHandle() as mock_server: try: - subprocess.run([test], timeout=6 * 60, check=True) + subprocess.run(cmd, timeout=6 * 60, check=True) except Exception as exc: print(f"Protocol test {test} failed with exception: {exc}") self.fail = True @@ -126,7 +274,7 @@ def _collect_tests(self): self.fail = True else: print(f"Found protocol tests {test_protocol}") - all_tests.append(test_protocol_exe) + all_tests.append((test_protocol_exe, test_protocol, test_group)) return all_tests