Skip to content

Commit 0e0a756

Browse files
zhongkechenzhongkechen
authored
add a new async child context performance test (#186)
* add a new async child context performance test * repeat multiple times for performance tests * assert the execution time for performance tests * adjust the format * clean up code * clean up code * format the output * format the output # Conflicts: # examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java
1 parent 9ce6d95 commit 0e0a756

6 files changed

Lines changed: 264 additions & 37 deletions

File tree

‎examples/src/main/java/software/amazon/lambda/durable/examples/ManyAsyncChildContextExample.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package software.amazon.lambda.durable.examples;
4+
5+
import java.time.Duration;
6+
import java.util.ArrayList;
7+
import java.util.concurrent.TimeUnit;
8+
import software.amazon.lambda.durable.DurableContext;
9+
import software.amazon.lambda.durable.DurableFuture;
10+
import software.amazon.lambda.durable.DurableHandler;
11+
12+
/**
13+
* Performance test example demonstrating concurrent async child contexts.
14+
*
15+
* <p>This example tests the SDK's ability to handle many concurrent operations:
16+
*
17+
* <ul>
18+
* <li>Creates async child context in a loop
19+
* <li>Each child context performs a simple computation in a step
20+
* <li>All results are collected using {@link DurableFuture#allOf}
21+
* </ul>
22+
*/
23+
public class ManyAsyncChildContextExample
24+
extends DurableHandler<ManyAsyncChildContextExample.Input, ManyAsyncChildContextExample.Output> {
25+
26+
public record Input(int multiplier, int steps) {}
27+
28+
public record Output(long result, long executionTimeMs, long replayTimeMs) {}
29+
30+
@Override
31+
public Output handleRequest(Input input, DurableContext context) {
32+
var startTime = System.nanoTime();
33+
var multiplier = input.multiplier();
34+
var steps = input.steps();
35+
var logger = context.getLogger();
36+
37+
logger.info("Starting {} async child context with multiplier {}", steps, multiplier);
38+
39+
// Create async steps
40+
var futures = new ArrayList<DurableFuture<Integer>>(steps);
41+
for (var i = 0; i < steps; i++) {
42+
var index = i;
43+
var future = context.runInChildContextAsync("child-" + i, Integer.class, childCtx -> {
44+
// create a step inside the child context, which doubles the number of threads
45+
return childCtx.step("compute-" + index, Integer.class, stepCtx -> index * multiplier);
46+
});
47+
futures.add(future);
48+
}
49+
50+
logger.info("All {} async child context created, collecting results", steps);
51+
52+
// Collect all results using allOf
53+
var results = DurableFuture.allOf(futures);
54+
var totalSum = results.stream().mapToInt(Integer::intValue).sum();
55+
56+
// checkpoint the executionTime so that we can have the same value when replay
57+
var executionTimeMs = context.step(
58+
"execution-time", Long.class, stepCtx -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime));
59+
logger.info(
60+
"Completed {} child context, total sum: {}, execution time: {}ms", steps, totalSum, executionTimeMs);
61+
62+
// Wait 2 seconds to test replay
63+
context.wait("post-compute-wait", Duration.ofSeconds(2));
64+
65+
var replayTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
66+
67+
return new Output(totalSum, executionTimeMs, replayTimeMs);
68+
}
69+
}

‎examples/src/main/java/software/amazon/lambda/durable/examples/ManyAsyncStepsExample.java‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,28 +20,30 @@
2020
* <li>All results are collected using {@link DurableFuture#allOf}
2121
* </ul>
2222
*/
23-
public class ManyAsyncStepsExample extends DurableHandler<ManyAsyncStepsExample.Input, String> {
23+
public class ManyAsyncStepsExample extends DurableHandler<ManyAsyncStepsExample.Input, ManyAsyncStepsExample.Output> {
2424

25-
private static final int STEP_COUNT = 500;
25+
public record Input(int multiplier, int steps) {}
2626

27-
public record Input(int multiplier) {}
27+
public record Output(long result, long executionTimeMs, long replayTimeMs) {}
2828

2929
@Override
30-
public String handleRequest(Input input, DurableContext context) {
30+
public Output handleRequest(Input input, DurableContext context) {
3131
var startTime = System.nanoTime();
32-
var multiplier = input.multiplier() > 0 ? input.multiplier() : 1;
32+
var multiplier = input.multiplier();
33+
var steps = input.steps();
34+
var logger = context.getLogger();
3335

34-
context.getLogger().info("Starting {} async steps with multiplier {}", STEP_COUNT, multiplier);
36+
logger.info("Starting {} async steps with multiplier {}", steps, multiplier);
3537

3638
// Create async steps
37-
var futures = new ArrayList<DurableFuture<Integer>>(STEP_COUNT);
38-
for (var i = 0; i < STEP_COUNT; i++) {
39+
var futures = new ArrayList<DurableFuture<Integer>>(steps);
40+
for (var i = 0; i < steps; i++) {
3941
var index = i;
4042
var future = context.stepAsync("compute-" + i, Integer.class, stepCtx -> index * multiplier);
4143
futures.add(future);
4244
}
4345

44-
context.getLogger().info("All {} async steps created, collecting results", STEP_COUNT);
46+
logger.info("All {} async steps created, collecting results", steps);
4547

4648
// Collect all results using allOf
4749
var results = DurableFuture.allOf(futures);
@@ -50,16 +52,13 @@ public String handleRequest(Input input, DurableContext context) {
5052
// checkpoint the executionTime so that we can have the same value when replay
5153
var executionTimeMs = context.step(
5254
"execution-time", Long.class, stepCtx -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime));
53-
context.getLogger()
54-
.info("Completed {} steps, total sum: {}, execution time: {}ms", STEP_COUNT, totalSum, executionTimeMs);
55+
logger.info("Completed {} steps, total sum: {}, execution time: {}ms", steps, totalSum, executionTimeMs);
5556

56-
// Wait 10 seconds to test replay
57-
context.wait("post-compute-wait", Duration.ofSeconds(10));
57+
// Wait 2 seconds to test replay
58+
context.wait("post-compute-wait", Duration.ofSeconds(2));
5859

5960
var replayTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
6061

61-
return String.format(
62-
"Completed %d async steps. Sum: %d, Execution Time: %dms, Replay Time: %dms",
63-
STEP_COUNT, totalSum, executionTimeMs, replayTimeMs);
62+
return new Output(totalSum, executionTimeMs, replayTimeMs);
6463
}
6564
}

‎examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java‎

Lines changed: 73 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import org.junit.jupiter.api.BeforeAll;
1212
import org.junit.jupiter.api.Test;
1313
import org.junit.jupiter.api.condition.EnabledIf;
14+
import org.junit.jupiter.params.ParameterizedTest;
15+
import org.junit.jupiter.params.provider.CsvSource;
1416
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
1517
import software.amazon.awssdk.core.SdkBytes;
1618
import software.amazon.awssdk.services.lambda.LambdaClient;
@@ -22,6 +24,7 @@
2224

2325
@EnabledIf("isEnabled")
2426
class CloudBasedIntegrationTest {
27+
private static final int PERFORMANCE_TEST_REPEAT = 3;
2528

2629
private static String account;
2730
private static String region;
@@ -462,22 +465,78 @@ void testChildContextExample() {
462465
assertNotNull(runner.getOperation("shipping-estimate"));
463466
}
464467

465-
@Test
466-
void testManyAsyncStepsExample() {
467-
var runner = CloudDurableTestRunner.create(
468-
arn("many-async-steps-example"), ManyAsyncStepsExample.Input.class, String.class);
469-
var result = runner.run(new ManyAsyncStepsExample.Input(2));
468+
@ParameterizedTest
469+
@CsvSource({"100, 1000, 10", "500, 2000, 20", "1000, 3000, 30"})
470+
void testManyAsyncStepsExample(int steps, long maxExecutionTime, long maxReplayTime) {
471+
long minimalExecutionTimeMs = Long.MAX_VALUE;
472+
long minimalReplayTimeMs = Long.MAX_VALUE;
473+
for (var i = 0; i < PERFORMANCE_TEST_REPEAT; i++) {
474+
var runner = CloudDurableTestRunner.create(
475+
arn("many-async-steps-example"),
476+
ManyAsyncStepsExample.Input.class,
477+
ManyAsyncStepsExample.Output.class);
478+
var result = runner.run(new ManyAsyncStepsExample.Input(2, steps));
479+
480+
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
481+
482+
var finalResult = result.getResult(ManyAsyncStepsExample.Output.class);
483+
System.out.printf("ManyAsyncStepsExample result (%d steps): %s\n", steps, finalResult);
484+
assertNotNull(finalResult);
485+
assertEquals((long) steps * (steps - 1), finalResult.result()); // Sum of 0..steps * 2
486+
487+
// Verify some operations are tracked
488+
assertNotNull(runner.getOperation("compute-0"));
489+
assertNotNull(runner.getOperation("compute-" + (steps - 1)));
490+
491+
if (finalResult.executionTimeMs() < minimalExecutionTimeMs) {
492+
minimalExecutionTimeMs = finalResult.executionTimeMs();
493+
}
494+
495+
if (finalResult.replayTimeMs() < minimalReplayTimeMs) {
496+
minimalReplayTimeMs = finalResult.replayTimeMs();
497+
}
498+
}
470499

471-
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
500+
assertTrue(minimalReplayTimeMs < maxReplayTime);
501+
assertTrue(minimalExecutionTimeMs < maxExecutionTime);
502+
}
472503

473-
var finalResult = result.getResult(String.class);
474-
System.out.println("ManyAsyncStepsExample result: " + finalResult);
475-
assertNotNull(finalResult);
476-
assertTrue(finalResult.contains("500 async steps"));
477-
assertTrue(finalResult.contains("249500")); // Sum of 0..499 * 2
504+
@ParameterizedTest
505+
// OOM if it creates 1000 child contexts
506+
@CsvSource({"100, 1500, 10", "500, 3000, 20"})
507+
void testManyAsyncChildContextExample(int steps, long maxExecutionTime, long maxReplayTime) {
508+
long minimalExecutionTimeMs = Long.MAX_VALUE;
509+
long minimalReplayTimeMs = Long.MAX_VALUE;
510+
for (var i = 0; i < PERFORMANCE_TEST_REPEAT; i++) {
511+
var runner = CloudDurableTestRunner.create(
512+
arn("many-async-child-context-example"),
513+
ManyAsyncChildContextExample.Input.class,
514+
ManyAsyncChildContextExample.Output.class);
515+
var result = runner.run(new ManyAsyncChildContextExample.Input(2, steps));
516+
517+
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
518+
519+
var finalResult = result.getResult(ManyAsyncChildContextExample.Output.class);
520+
System.out.printf("ManyAsyncChildContextExample result (%d child contexts): %s\n", steps, finalResult);
521+
assertNotNull(finalResult);
522+
assertEquals((long) steps * (steps - 1), finalResult.result()); // Sum of 0..steps * 2
523+
524+
// Verify some operations are tracked
525+
assertNotNull(runner.getOperation("compute-0"));
526+
assertNotNull(runner.getOperation("compute-" + (steps - 1)));
527+
assertNotNull(runner.getOperation("child-0"));
528+
assertNotNull(runner.getOperation("child-" + (steps - 1)));
529+
530+
if (finalResult.executionTimeMs() < minimalExecutionTimeMs) {
531+
minimalExecutionTimeMs = finalResult.executionTimeMs();
532+
}
533+
534+
if (finalResult.replayTimeMs() < minimalReplayTimeMs) {
535+
minimalReplayTimeMs = finalResult.replayTimeMs();
536+
}
537+
}
478538

479-
// Verify some operations are tracked
480-
assertNotNull(runner.getOperation("compute-0"));
481-
assertNotNull(runner.getOperation("compute-499"));
539+
assertTrue(minimalReplayTimeMs < maxReplayTime);
540+
assertTrue(minimalExecutionTimeMs < maxExecutionTime);
482541
}
483542
}

‎examples/src/test/java/software/amazon/lambda/durable/examples/ManyAsyncChildContextExampleTest.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package software.amazon.lambda.durable.examples;
4+
5+
import static org.junit.jupiter.api.Assertions.assertEquals;
6+
import static org.junit.jupiter.api.Assertions.assertNotNull;
7+
8+
import org.junit.jupiter.api.Test;
9+
import software.amazon.lambda.durable.model.ExecutionStatus;
10+
import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
11+
12+
class ManyAsyncChildContextExampleTest {
13+
14+
@Test
15+
void testManyAsyncSteps() {
16+
var handler = new ManyAsyncChildContextExample();
17+
var runner = LocalDurableTestRunner.create(ManyAsyncChildContextExample.Input.class, handler);
18+
19+
var input = new ManyAsyncChildContextExample.Input(2, 500);
20+
var result = runner.runUntilComplete(input);
21+
22+
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
23+
24+
var output = result.getResult(ManyAsyncStepsExample.Output.class);
25+
assertNotNull(output);
26+
27+
// Sum of 0..499 * 2 = 499 * 500 / 2 * 2 = 249500
28+
assertEquals(249500, output.result());
29+
}
30+
31+
@Test
32+
void testManyAsyncStepsWithDefaultMultiplier() {
33+
var handler = new ManyAsyncChildContextExample();
34+
var runner = LocalDurableTestRunner.create(ManyAsyncChildContextExample.Input.class, handler);
35+
36+
var input = new ManyAsyncChildContextExample.Input(1, 500);
37+
var result = runner.runUntilComplete(input);
38+
39+
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
40+
41+
// Sum of 0..499 = 499 * 500 / 2 = 124750
42+
assertEquals(
43+
124750,
44+
result.getResult(ManyAsyncChildContextExample.Output.class).result());
45+
}
46+
47+
@Test
48+
void testOperationsAreTracked() {
49+
var handler = new ManyAsyncChildContextExample();
50+
var runner = LocalDurableTestRunner.create(ManyAsyncChildContextExample.Input.class, handler);
51+
52+
var result = runner.runUntilComplete(new ManyAsyncChildContextExample.Input(1, 500));
53+
54+
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
55+
56+
// Verify some operations are tracked
57+
assertNotNull(result.getOperation("compute-0"));
58+
assertNotNull(result.getOperation("compute-499"));
59+
assertNotNull(result.getOperation("compute-250"));
60+
61+
assertNotNull(result.getOperation("child-0"));
62+
assertNotNull(result.getOperation("child-499"));
63+
assertNotNull(result.getOperation("child-250"));
64+
}
65+
}

‎examples/src/test/java/software/amazon/lambda/durable/examples/ManyAsyncStepsExampleTest.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,40 @@ void testManyAsyncSteps() {
1515
var handler = new ManyAsyncStepsExample();
1616
var runner = LocalDurableTestRunner.create(ManyAsyncStepsExample.Input.class, handler);
1717

18-
var input = new ManyAsyncStepsExample.Input(2);
18+
var input = new ManyAsyncStepsExample.Input(2, 500);
1919
var result = runner.runUntilComplete(input);
2020

2121
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
2222

23-
var output = result.getResult(String.class);
23+
var output = result.getResult(ManyAsyncStepsExample.Output.class);
2424
assertNotNull(output);
25-
assertTrue(output.contains("500 async steps"));
2625

2726
// Sum of 0..499 * 2 = 499 * 500 / 2 * 2 = 249500
28-
assertTrue(output.contains("249500"));
27+
assertEquals(
28+
249500, result.getResult(ManyAsyncStepsExample.Output.class).result());
2929
}
3030

3131
@Test
3232
void testManyAsyncStepsWithDefaultMultiplier() {
3333
var handler = new ManyAsyncStepsExample();
3434
var runner = LocalDurableTestRunner.create(ManyAsyncStepsExample.Input.class, handler);
3535

36-
var input = new ManyAsyncStepsExample.Input(1);
36+
var input = new ManyAsyncStepsExample.Input(1, 500);
3737
var result = runner.runUntilComplete(input);
3838

3939
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
4040

4141
// Sum of 0..499 = 499 * 500 / 2 = 124750
42-
assertTrue(result.getResult(String.class).contains("124750"));
42+
assertEquals(
43+
124750, result.getResult(ManyAsyncStepsExample.Output.class).result());
4344
}
4445

4546
@Test
4647
void testOperationsAreTracked() {
4748
var handler = new ManyAsyncStepsExample();
4849
var runner = LocalDurableTestRunner.create(ManyAsyncStepsExample.Input.class, handler);
4950

50-
var result = runner.runUntilComplete(new ManyAsyncStepsExample.Input(1));
51+
var result = runner.runUntilComplete(new ManyAsyncStepsExample.Input(1, 500));
5152

5253
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
5354

‎examples/template.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,31 @@ Resources:
443443
DockerContext: ../
444444
DockerTag: durable-examples
445445

446+
ManyAsyncChildContextExampleFunction:
447+
Type: AWS::Serverless::Function
448+
Properties:
449+
PackageType: Image
450+
FunctionName: !Join
451+
- ''
452+
- - 'many-async-child-context-example'
453+
- !Ref FunctionNameSuffix
454+
ImageConfig:
455+
Command: ["software.amazon.lambda.durable.examples.ManyAsyncChildContextExample::handleRequest"]
456+
DurableConfig:
457+
ExecutionTimeout: 300
458+
RetentionPeriodInDays: 7
459+
Policies:
460+
- Statement:
461+
- Effect: Allow
462+
Action:
463+
- lambda:CheckpointDurableExecutions
464+
- lambda:GetDurableExecutionState
465+
Resource: !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:many-async-child-context-example${FunctionNameSuffix}"
466+
Metadata:
467+
Dockerfile: !Ref DockerFile
468+
DockerContext: ../
469+
DockerTag: durable-examples
470+
446471
Outputs:
447472
NoopExampleFunction:
448473
Description: Noop Example Function ARN
@@ -579,3 +604,12 @@ Outputs:
579604
WaitAsyncExampleFunctionName:
580605
Description: Wait Async Example Function Name
581606
Value: !Ref WaitAsyncExampleFunction
607+
608+
ManyAsyncChildContextExampleFunction:
609+
Description: Many Async Child Context Example Function ARN
610+
Value: !GetAtt ManyAsyncChildContextExampleFunction.Arn
611+
612+
ManyAsyncChildContextExampleFunctionName:
613+
Description: Many Async Child Context Example Function Name
614+
Value: !Ref ManyAsyncChildContextExampleFunction
615+

0 commit comments

Comments
 (0)