Skip to content

Commit eca21f8

Browse files
alextwoodsdavidh44
alextwoods
and
davidh44
authored
Add expectContinueThresholdInBytes config to S3 (#6864)
* Add expectContinueThresholdInBytes config to S3 * Add more docs * Improve docs * PR feedback * Bump minor version * Fix Expect100ContinueHeaderTest * Revert "Bump minor version" This reverts commit 037d3fd. --------- Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com>
1 parent c013225 commit eca21f8

6 files changed

Lines changed: 242 additions & 14 deletions

File tree

‎.changes/next-release/feature-AmazonS3-08f1e6b.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "Amazon S3",
4+
"contributor": "",
5+
"description": "Add configurable `expectContinueThresholdInBytes` to S3Configuration (default 1 MB). The Expect: 100-continue header is now only added to PutObject and UploadPart requests when the content-length meets or exceeds the threshold, reducing latency overhead for small uploads."
6+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Configuration.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,19 @@ public final class S3Configuration implements ServiceConfiguration, ToCopyableBu
7777
*/
7878
private static final boolean DEFAULT_EXPECT_CONTINUE_ENABLED = true;
7979

80+
/**
81+
* The default minimum content-length in bytes at which the {@code Expect: 100-continue} header is added.
82+
* Requests with a content-length below this threshold will not include the header.
83+
*/
84+
private static final long DEFAULT_EXPECT_CONTINUE_THRESHOLD_IN_BYTES = 1_048_576L;
85+
8086
private final FieldWithDefault<Boolean> pathStyleAccessEnabled;
8187
private final FieldWithDefault<Boolean> accelerateModeEnabled;
8288
private final FieldWithDefault<Boolean> dualstackEnabled;
8389
private final FieldWithDefault<Boolean> checksumValidationEnabled;
8490
private final FieldWithDefault<Boolean> chunkedEncodingEnabled;
8591
private final FieldWithDefault<Boolean> expectContinueEnabled;
92+
private final FieldWithDefault<Long> expectContinueThresholdInBytes;
8693
private final Boolean useArnRegionEnabled;
8794
private final Boolean multiRegionEnabled;
8895
private final FieldWithDefault<Supplier<ProfileFile>> profileFile;
@@ -97,6 +104,13 @@ private S3Configuration(DefaultS3ServiceConfigurationBuilder builder) {
97104
this.chunkedEncodingEnabled = FieldWithDefault.create(builder.chunkedEncodingEnabled, DEFAULT_CHUNKED_ENCODING_ENABLED);
98105
this.expectContinueEnabled = FieldWithDefault.create(builder.expectContinueEnabled,
99106
DEFAULT_EXPECT_CONTINUE_ENABLED);
107+
this.expectContinueThresholdInBytes = FieldWithDefault.create(builder.expectContinueThresholdInBytes,
108+
DEFAULT_EXPECT_CONTINUE_THRESHOLD_IN_BYTES);
109+
if (this.expectContinueThresholdInBytes.value() < 0) {
110+
throw new IllegalArgumentException(
111+
"expectContinueThresholdInBytes must not be negative, but was: "
112+
+ this.expectContinueThresholdInBytes.value());
113+
}
100114
this.profileFile = FieldWithDefault.create(builder.profileFile, ProfileFile::defaultProfileFile);
101115
this.profileName = FieldWithDefault.create(builder.profileName,
102116
ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow());
@@ -247,6 +261,26 @@ public boolean expectContinueEnabled() {
247261
return expectContinueEnabled.value();
248262
}
249263

264+
/**
265+
* Returns the minimum content-length in bytes at which the {@code Expect: 100-continue} header is added to
266+
* {@link PutObjectRequest} and {@link UploadPartRequest}. Requests with a content-length below this threshold
267+
* will not include the header.
268+
* <p>
269+
* The default value is 1048576 bytes (1 MB).
270+
* <p>
271+
* <b>Note:</b> When using the {@code ApacheHttpClient} (Apache 4), the Apache 4 client also independently adds the
272+
* {@code Expect: 100-continue} header by default without any threshold via its own {@code expectContinueEnabled}
273+
* setting. To benefit from the `expectContinueThresholdInBytes` you must disable {@code expectContinueEnabled}
274+
* on the Apache4 HTTP client builder using {@code ApacheHttpClient.builder().expectContinueEnabled(false)}.
275+
* This does NOT apply to the {@code Apache5HttpClient} which defaults {@code expectContinueEnabled} to false.
276+
*
277+
* @return The threshold in bytes.
278+
* @see S3Configuration.Builder#expectContinueThresholdInBytes(Long)
279+
*/
280+
public long expectContinueThresholdInBytes() {
281+
return expectContinueThresholdInBytes.value();
282+
}
283+
250284
/**
251285
* Returns whether the client is allowed to make cross-region calls when an S3 Access Point ARN has a different
252286
* region to the one configured on the client.
@@ -278,6 +312,7 @@ public Builder toBuilder() {
278312
.checksumValidationEnabled(checksumValidationEnabled.valueOrNullIfDefault())
279313
.chunkedEncodingEnabled(chunkedEncodingEnabled.valueOrNullIfDefault())
280314
.expectContinueEnabled(expectContinueEnabled.valueOrNullIfDefault())
315+
.expectContinueThresholdInBytes(expectContinueThresholdInBytes.valueOrNullIfDefault())
281316
.useArnRegionEnabled(useArnRegionEnabled)
282317
.profileFile(profileFile.valueOrNullIfDefault())
283318
.profileName(profileName.valueOrNullIfDefault());
@@ -407,6 +442,32 @@ public interface Builder extends CopyableBuilder<Builder, S3Configuration> {
407442
*/
408443
Builder expectContinueEnabled(Boolean expectContinueEnabled);
409444

445+
Long expectContinueThresholdInBytes();
446+
447+
/**
448+
* Option to configure the minimum content-length in bytes at which the {@code Expect: 100-continue} header
449+
* is added to {@link PutObjectRequest} and {@link UploadPartRequest}. Requests with a content-length below
450+
* this threshold will not include the header, reducing latency for small uploads where the round-trip cost
451+
* of the 100-continue handshake outweighs the benefit.
452+
* <p>
453+
* The default value is 1048576 bytes (1 MB). Setting this to 0 restores the pre-threshold behavior where
454+
* the header is added for all non-zero content-length requests.
455+
* <p>
456+
* This setting only takes effect when {@link #expectContinueEnabled(Boolean)} is {@code true} (the default).
457+
* <p>
458+
* When content length is not known, the {@code Expect: 100-continue} header will always be added
459+
* when {@link #expectContinueEnabled(Boolean)} is {@code true}.
460+
* <p>
461+
* <b>Note:</b> When using the {@code ApacheHttpClient} (Apache 4), the Apache 4 client also independently adds the
462+
* {@code Expect: 100-continue} header by default via its own {@code expectContinueEnabled} setting. This threshold
463+
* only controls the SDK's own header addition; it does not affect the Apache client's behavior.
464+
*
465+
* @param expectContinueThresholdInBytes The threshold in bytes, or {@code null} to use the default (1048576).
466+
* @return This builder for method chaining.
467+
* @see S3Configuration#expectContinueThresholdInBytes()
468+
*/
469+
Builder expectContinueThresholdInBytes(Long expectContinueThresholdInBytes);
470+
410471
Boolean useArnRegionEnabled();
411472

412473
/**
@@ -476,6 +537,7 @@ static final class DefaultS3ServiceConfigurationBuilder implements Builder {
476537
private Boolean checksumValidationEnabled;
477538
private Boolean chunkedEncodingEnabled;
478539
private Boolean expectContinueEnabled;
540+
private Long expectContinueThresholdInBytes;
479541
private Boolean useArnRegionEnabled;
480542
private Boolean multiRegionEnabled;
481543
private Supplier<ProfileFile> profileFile;
@@ -571,6 +633,21 @@ public void setExpectContinueEnabled(Boolean expectContinueEnabled) {
571633
expectContinueEnabled(expectContinueEnabled);
572634
}
573635

636+
@Override
637+
public Long expectContinueThresholdInBytes() {
638+
return expectContinueThresholdInBytes;
639+
}
640+
641+
@Override
642+
public Builder expectContinueThresholdInBytes(Long expectContinueThresholdInBytes) {
643+
this.expectContinueThresholdInBytes = expectContinueThresholdInBytes;
644+
return this;
645+
}
646+
647+
public void setExpectContinueThresholdInBytes(Long expectContinueThresholdInBytes) {
648+
expectContinueThresholdInBytes(expectContinueThresholdInBytes);
649+
}
650+
574651
@Override
575652
public Boolean useArnRegionEnabled() {
576653
return useArnRegionEnabled;

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/StreamingRequestInterceptor.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,22 +55,24 @@ private boolean shouldAddExpectContinueHeader(Context.ModifyHttpRequest context,
5555
return false;
5656
}
5757

58-
if (isExpect100ContinueDisabled(executionAttributes)) {
58+
S3Configuration s3Config = getS3Configuration(executionAttributes);
59+
60+
if (s3Config != null && !s3Config.expectContinueEnabled()) {
5961
return false;
6062
}
6163

64+
long threshold = s3Config != null ? s3Config.expectContinueThresholdInBytes()
65+
: 0L;
66+
6267
return getContentLengthHeader(context.httpRequest())
6368
.map(Long::parseLong)
64-
.map(length -> length != 0L)
69+
.map(length -> length >= threshold && length != 0L)
6570
.orElse(true);
6671
}
6772

68-
private boolean isExpect100ContinueDisabled(ExecutionAttributes executionAttributes) {
73+
private S3Configuration getS3Configuration(ExecutionAttributes executionAttributes) {
6974
ServiceConfiguration serviceConfig = executionAttributes.getAttribute(SdkExecutionAttribute.SERVICE_CONFIG);
70-
if (serviceConfig instanceof S3Configuration) {
71-
return !((S3Configuration) serviceConfig).expectContinueEnabled();
72-
}
73-
return false;
75+
return serviceConfig instanceof S3Configuration ? (S3Configuration) serviceConfig : null;
7476
}
7577

7678
/**

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/S3ConfigurationTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package software.amazon.awssdk.services.s3;
1717

1818
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
1920
import static software.amazon.awssdk.profiles.ProfileFileSystemSetting.AWS_CONFIG_FILE;
2021
import static software.amazon.awssdk.services.s3.S3SystemSetting.AWS_S3_DISABLE_MULTIREGION_ACCESS_POINTS;
2122
import static software.amazon.awssdk.services.s3.S3SystemSetting.AWS_S3_USE_ARN_REGION;
@@ -47,6 +48,7 @@ public void createConfiguration_minimal() {
4748
assertThat(config.pathStyleAccessEnabled()).isFalse();
4849
assertThat(config.useArnRegionEnabled()).isFalse();
4950
assertThat(config.expectContinueEnabled()).isTrue();
51+
assertThat(config.expectContinueThresholdInBytes()).isEqualTo(1048576L);
5052
}
5153

5254
@Test
@@ -116,5 +118,55 @@ public void useArnRegionEnabled_enabledInCProfile_shouldResolveToConfigCorrectly
116118
assertThat(config.useArnRegionEnabled()).isEqualTo(false);
117119
}
118120

121+
// -----------------------------------------------------------------------
122+
// expectContinueThresholdInBytes
123+
// -----------------------------------------------------------------------
124+
125+
@Test
126+
public void expectContinueThresholdInBytes_defaultValue_is1MB() {
127+
S3Configuration config = S3Configuration.builder().build();
128+
assertThat(config.expectContinueThresholdInBytes()).isEqualTo(1048576L);
129+
}
130+
131+
@Test
132+
public void expectContinueThresholdInBytes_customValue_isPreserved() {
133+
S3Configuration config = S3Configuration.builder()
134+
.expectContinueThresholdInBytes(2_097_152L)
135+
.build();
136+
assertThat(config.expectContinueThresholdInBytes()).isEqualTo(2_097_152L);
137+
}
138+
139+
@Test
140+
public void expectContinueThresholdInBytes_toBuilder_preservesUserSetValue() {
141+
S3Configuration config = S3Configuration.builder()
142+
.expectContinueThresholdInBytes(512L)
143+
.build();
144+
S3Configuration rebuilt = config.toBuilder().build();
145+
assertThat(rebuilt.expectContinueThresholdInBytes()).isEqualTo(512L);
146+
}
147+
148+
@Test
149+
public void expectContinueThresholdInBytes_toBuilder_returnsNullForDefault() {
150+
S3Configuration config = S3Configuration.builder().build();
151+
S3Configuration.Builder builder = config.toBuilder();
152+
assertThat(builder.expectContinueThresholdInBytes()).isNull();
153+
}
154+
155+
@Test
156+
public void expectContinueThresholdInBytes_negativeValue_throwsException() {
157+
assertThatThrownBy(() -> S3Configuration.builder()
158+
.expectContinueThresholdInBytes(-1L)
159+
.build())
160+
.isInstanceOf(IllegalArgumentException.class)
161+
.hasMessageContaining("expectContinueThresholdInBytes must not be negative");
162+
}
163+
164+
@Test
165+
public void expectContinueThresholdInBytes_zeroValue_isAccepted() {
166+
S3Configuration config = S3Configuration.builder()
167+
.expectContinueThresholdInBytes(0L)
168+
.build();
169+
assertThat(config.expectContinueThresholdInBytes()).isEqualTo(0L);
170+
}
119171

120172
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/functionaltests/Expect100ContinueHeaderTest.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,9 @@ void setup(WireMockRuntimeInfo wmRuntimeInfo) {
9696
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
9797
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
9898
.forcePathStyle(true)
99+
.serviceConfiguration(S3Configuration.builder()
100+
.expectContinueThresholdInBytes(0L)
101+
.build())
99102
.credentialsProvider(staticCredentials())
100103
.build();
101104

@@ -106,6 +109,9 @@ void setup(WireMockRuntimeInfo wmRuntimeInfo) {
106109
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
107110
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
108111
.forcePathStyle(true)
112+
.serviceConfiguration(S3Configuration.builder()
113+
.expectContinueThresholdInBytes(0L)
114+
.build())
109115
.credentialsProvider(staticCredentials())
110116
.build();
111117
}
@@ -290,6 +296,7 @@ private static Stream<Arguments> expectContinueConfigProvider() {
290296
.build();
291297
S3Configuration enabledConfig = S3Configuration.builder()
292298
.expectContinueEnabled(true)
299+
.expectContinueThresholdInBytes(0L)
293300
.build();
294301

295302
return Stream.of(
@@ -381,7 +388,10 @@ private S3Client buildSyncClient(String clientType, WireMockRuntimeInfo wmInfo,
381388
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
382389
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
383390
.forcePathStyle(true)
384-
.serviceConfiguration(config != null ? config : S3Configuration.builder().build())
391+
.serviceConfiguration(config != null ? config
392+
: S3Configuration.builder()
393+
.expectContinueThresholdInBytes(0L)
394+
.build())
385395
.credentialsProvider(staticCredentials())
386396
.build();
387397
}
@@ -406,7 +416,10 @@ private S3AsyncClient buildAsyncClient(String clientType, WireMockRuntimeInfo wm
406416
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
407417
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
408418
.forcePathStyle(true)
409-
.serviceConfiguration(config != null ? config : S3Configuration.builder().build())
419+
.serviceConfiguration(config != null ? config
420+
: S3Configuration.builder()
421+
.expectContinueThresholdInBytes(0L)
422+
.build())
410423
.credentialsProvider(staticCredentials())
411424
.build();
412425
}

0 commit comments

Comments
 (0)