Skip to content

Commit 47b72f3

Browse files
author
Frank
committed
sync
1 parent f24df89 commit 47b72f3

File tree

12 files changed

+789
-95
lines changed

12 files changed

+789
-95
lines changed

‎cli.ts‎

Lines changed: 15 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -41,21 +41,15 @@ const cli = yargs(hideBin(process.argv))
4141
"$0 opencode --model opencode/claude-sonnet-4-5 --eval DataDog/datadog-lambda-python@93d4a07..d776378 --output results.json",
4242
],
4343
])
44-
.fail((msg) => {
45-
console.error(msg);
46-
process.exit(1);
47-
})
4844
.strict();
4945

5046
cli.command(
5147
"generate",
5248
"Generate dataset for all evaluations",
5349
async (yargs) =>
5450
yargs.example([["orvl generate", "Generate dataset for all evaluations"]]),
55-
async ({ eval: evalId }) => {
56-
const logger = Logger.create();
57-
logger.log(`Generating dataset...`);
58-
51+
async () => {
52+
const logger = Logger.create("[generate]");
5953
await Eval.generate({ logger });
6054
},
6155
);
@@ -107,7 +101,7 @@ cli.command(
107101
const evals = await Eval.load();
108102
const agent = getAgent(agentName);
109103
const model = getModel(agent, modelFilter);
110-
const evalDef = getEval(evals, evalId);
104+
const ev = getEval(evals, evalId);
111105
const logger = Logger.create(`[model ${model}]`);
112106

113107
// Run episodes
@@ -116,14 +110,11 @@ cli.command(
116110
const index = offset + 1;
117111
const childLogger = logger.child(`[episode ${index}/${episodes}]`);
118112
childLogger.log(`Starting episode with ${timeoutMins}min timeout...`);
119-
return withRetries(
120-
() => runEpisode(evalDef, agent, model, childLogger),
121-
{
122-
retries: 3,
123-
timeoutMs: timeoutMins * 60 * 1000,
124-
logger: childLogger,
125-
},
126-
).then((result) => ({ index, ...result }));
113+
return withRetries(() => runEpisode(ev, agent, model, childLogger), {
114+
retries: 3,
115+
timeoutMs: timeoutMins * 60 * 1000,
116+
logger: childLogger,
117+
}).then((result) => ({ index, ...result }));
127118
}),
128119
);
129120

@@ -174,15 +165,11 @@ cli.command(
174165
});
175166

176167
// Generate summary from all episodes' actions
177-
const summary = await generateActionsSummary(
178-
evalDef,
179-
model,
180-
episodesActions,
181-
);
168+
const summary = await generateActionsSummary(ev, model, episodesActions);
182169

183170
const evaluationResult = summarizeAggregation(
184171
agent.name,
185-
evalDef,
172+
ev,
186173
model,
187174
aggregatedInputs,
188175
episodeExports,
@@ -229,13 +216,13 @@ function getModel(agent: Agent.Registration, modelFilter: string) {
229216
}
230217

231218
function getEval(evals: Eval.Instance[], evalId: string) {
232-
const evalDef = evals.find((ev) => ev.id === evalId);
233-
if (!evalDef) throw new Error(`Eval ${evalId} was not found.`);
234-
if (!evalDef.scores.length)
219+
const ev = evals.find((ev) => ev.id === evalId);
220+
if (!ev) throw new Error(`Eval ${evalId} was not found.`);
221+
if (!ev.scores.length)
235222
throw new Error(
236-
`Evaluation ${evalDef.repo} has no score assignments configured.`,
223+
`Evaluation ${ev.repo} has no score assignments configured.`,
237224
);
238-
return evalDef;
225+
return ev;
239226
}
240227

241228
async function runEpisode(

‎evals/datadog-lambda-python/diff.patch‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
diff --git a/datadog_lambda/metric.py b/datadog_lambda/metric.py
2+
index 73bbeca3..5df0812f 100644
3+
--- a/datadog_lambda/metric.py
4+
+++ b/datadog_lambda/metric.py
5+
@@ -214,6 +214,33 @@ def submit_errors_metric(lambda_context):
6+
submit_enhanced_metric("errors", lambda_context)
7+
8+
9+
+def submit_batch_item_failures_metric(response, lambda_context):
10+
+ """Submit aws.lambda.enhanced.batch_item_failures metric with the count of batch item failures
11+
+
12+
+ Args:
13+
+ response (dict): Lambda function response object
14+
+ lambda_context (object): Lambda context dict passed to the function by AWS
15+
+ """
16+
+ if not config.enhanced_metrics_enabled:
17+
+ logger.debug(
18+
+ "Not submitting batch_item_failures metric because enhanced metrics are disabled"
19+
+ )
20+
+ return
21+
+
22+
+ if not isinstance(response, dict):
23+
+ return
24+
+
25+
+ batch_item_failures = response.get("batchItemFailures")
26+
+ if batch_item_failures is not None and isinstance(batch_item_failures, list):
27+
+ lambda_metric(
28+
+ "aws.lambda.enhanced.batch_item_failures",
29+
+ len(batch_item_failures),
30+
+ timestamp=None,
31+
+ tags=get_enhanced_metrics_tags(lambda_context),
32+
+ force_async=True,
33+
+ )
34+
+
35+
+
36+
def submit_dynamodb_stream_type_metric(event):
37+
stream_view_type = (
38+
event.get("Records", [{}])[0].get("dynamodb", {}).get("StreamViewType")
39+
diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py
40+
index 0cbedd9f..8dbd7e35 100644
41+
--- a/datadog_lambda/wrapper.py
42+
+++ b/datadog_lambda/wrapper.py
43+
@@ -291,6 +291,10 @@ def _before(self, event, context):
44+
45+
def _after(self, event, context):
46+
try:
47+
+ from datadog_lambda.metric import submit_batch_item_failures_metric
48+
+
49+
+ submit_batch_item_failures_metric(self.response, context)
50+
+
51+
status_code = extract_http_status_code_tag(self.trigger_tags, self.response)
52+
53+
if self.span:
54+
diff --git a/tests/test_metric.py b/tests/test_metric.py
55+
index aa537d34..fe3df247 100644
56+
--- a/tests/test_metric.py
57+
+++ b/tests/test_metric.py
58+
@@ -12,6 +12,7 @@
59+
_select_metrics_handler,
60+
flush_stats,
61+
lambda_metric,
62+
+ submit_batch_item_failures_metric,
63+
)
64+
from datadog_lambda.tags import dd_lambda_layer_tag
65+
from datadog_lambda.thread_stats_writer import ThreadStatsWriter
66+
@@ -324,3 +325,80 @@ def decrypt(self, CiphertextBlob=None, EncryptionContext={}):
67+
mock_kms_client, MOCK_ENCRYPTED_API_KEY_BASE64
68+
)
69+
self.assertEqual(decrypted_key, EXPECTED_DECRYPTED_API_KEY)
70+
+
71+
+
72+
+class TestBatchItemFailuresMetric(unittest.TestCase):
73+
+ def setUp(self):
74+
+ patcher = patch("datadog_lambda.metric.lambda_metric")
75+
+ self.mock_lambda_metric = patcher.start()
76+
+ self.addCleanup(patcher.stop)
77+
+
78+
+ patcher = patch("datadog_lambda.config.Config.enhanced_metrics_enabled", True)
79+
+ self.mock_enhanced_metrics_enabled = patcher.start()
80+
+ self.addCleanup(patcher.stop)
81+
+
82+
+ def test_submit_batch_item_failures_with_failures(self):
83+
+ response = {
84+
+ "batchItemFailures": [
85+
+ {"itemIdentifier": "msg-1"},
86+
+ {"itemIdentifier": "msg-2"},
87+
+ {"itemIdentifier": "msg-3"},
88+
+ ]
89+
+ }
90+
+ context = unittest.mock.Mock()
91+
+
92+
+ with patch("datadog_lambda.metric.get_enhanced_metrics_tags") as mock_get_tags:
93+
+ mock_get_tags.return_value = ["tag1:value1"]
94+
+ submit_batch_item_failures_metric(response, context)
95+
+
96+
+ self.mock_lambda_metric.assert_called_once_with(
97+
+ "aws.lambda.enhanced.batch_item_failures",
98+
+ 3,
99+
+ timestamp=None,
100+
+ tags=["tag1:value1"],
101+
+ force_async=True,
102+
+ )
103+
+
104+
+ def test_submit_batch_item_failures_with_no_failures(self):
105+
+ response = {"batchItemFailures": []}
106+
+ context = unittest.mock.Mock()
107+
+
108+
+ with patch("datadog_lambda.metric.get_enhanced_metrics_tags") as mock_get_tags:
109+
+ mock_get_tags.return_value = ["tag1:value1"]
110+
+ submit_batch_item_failures_metric(response, context)
111+
+ self.mock_lambda_metric.assert_called_once_with(
112+
+ "aws.lambda.enhanced.batch_item_failures",
113+
+ 0,
114+
+ timestamp=None,
115+
+ tags=["tag1:value1"],
116+
+ force_async=True,
117+
+ )
118+
+
119+
+ def test_submit_batch_item_failures_with_no_field(self):
120+
+ response = {"statusCode": 200}
121+
+ context = unittest.mock.Mock()
122+
+ submit_batch_item_failures_metric(response, context)
123+
+ self.mock_lambda_metric.assert_not_called()
124+
+
125+
+ def test_submit_batch_item_failures_with_none_response(self):
126+
+ response = None
127+
+ context = unittest.mock.Mock()
128+
+ submit_batch_item_failures_metric(response, context)
129+
+ self.mock_lambda_metric.assert_not_called()
130+
+
131+
+ def test_submit_batch_item_failures_with_non_list_value(self):
132+
+ response = {"batchItemFailures": "invalid"}
133+
+ context = unittest.mock.Mock()
134+
+ submit_batch_item_failures_metric(response, context)
135+
+ self.mock_lambda_metric.assert_not_called()
136+
+
137+
+ @patch("datadog_lambda.config.Config.enhanced_metrics_enabled", False)
138+
+ def test_submit_batch_item_failures_enhanced_metrics_disabled(self):
139+
+ response = {
140+
+ "batchItemFailures": [
141+
+ {"itemIdentifier": "msg-1"},
142+
+ ]
143+
+ }
144+
+ context = unittest.mock.Mock()
145+
+ submit_batch_item_failures_metric(response, context)
146+
+ self.mock_lambda_metric.assert_not_called()
147+
diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py
148+
index fe7678ac..512a51f8 100644
149+
--- a/tests/test_wrapper.py
150+
+++ b/tests/test_wrapper.py
151+
@@ -899,3 +899,61 @@ def lambda_handler(event, context):
152+
assert response == expected_response
153+
assert len(LLMObs_enable_calls) == 1
154+
assert len(LLMObs_flush_calls) == 1
155+
+
156+
+
157+
+@patch("datadog_lambda.config.Config.trace_enabled", False)
158+
+def test_batch_item_failures_metric():
159+
+ with patch(
160+
+ "datadog_lambda.metric.submit_batch_item_failures_metric"
161+
+ ) as mock_submit:
162+
+
163+
+ @wrapper.datadog_lambda_wrapper
164+
+ def lambda_handler(event, context):
165+
+ return {
166+
+ "batchItemFailures": [
167+
+ {"itemIdentifier": "msg-1"},
168+
+ {"itemIdentifier": "msg-2"},
169+
+ ]
170+
+ }
171+
+
172+
+ lambda_handler({}, get_mock_context())
173+
+ mock_submit.assert_called_once()
174+
+ call_args = mock_submit.call_args[0]
175+
+ assert call_args[0] == {
176+
+ "batchItemFailures": [
177+
+ {"itemIdentifier": "msg-1"},
178+
+ {"itemIdentifier": "msg-2"},
179+
+ ]
180+
+ }
181+
+
182+
+
183+
+@patch("datadog_lambda.config.Config.trace_enabled", False)
184+
+def test_batch_item_failures_metric_no_failures():
185+
+ with patch(
186+
+ "datadog_lambda.metric.submit_batch_item_failures_metric"
187+
+ ) as mock_submit:
188+
+
189+
+ @wrapper.datadog_lambda_wrapper
190+
+ def lambda_handler(event, context):
191+
+ return {"batchItemFailures": []}
192+
+
193+
+ lambda_handler({}, get_mock_context())
194+
+ mock_submit.assert_called_once()
195+
+ call_args = mock_submit.call_args[0]
196+
+ assert call_args[0] == {"batchItemFailures": []}
197+
+
198+
+
199+
+@patch("datadog_lambda.config.Config.trace_enabled", False)
200+
+def test_batch_item_failures_metric_no_response():
201+
+ with patch(
202+
+ "datadog_lambda.metric.submit_batch_item_failures_metric"
203+
+ ) as mock_submit:
204+
+
205+
+ @wrapper.datadog_lambda_wrapper
206+
+ def lambda_handler(event, context):
207+
+ return None
208+
+
209+
+ lambda_handler({}, get_mock_context())
210+
+ mock_submit.assert_called_once()
211+
+ call_args = mock_submit.call_args[0]
212+
+ assert call_args[0] is None

‎evals/helix-db-cli-update-blocking/diff.patch‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
diff --git a/helix-cli/src/commands/update.rs b/helix-cli/src/commands/update.rs
2+
index 22269ae2a..137b73850 100644
3+
--- a/helix-cli/src/commands/update.rs
4+
+++ b/helix-cli/src/commands/update.rs
5+
@@ -4,6 +4,13 @@ use self_update::cargo_crate_version;
6+
use crate::utils::{print_error_with_hint, print_status, print_success};
7+
8+
pub async fn run(force: bool) -> Result<()> {
9+
+ // We're using the self_update crate which is very handy but doesn't support async.
10+
+ // Still, this is good enough, but because it panics in an async context we must
11+
+ // do a spawn_blocking
12+
+ tokio::task::spawn_blocking(move || run_sync(force)).await?
13+
+}
14+
+
15+
+fn run_sync(force: bool) -> Result<()> {
16+
print_status("UPDATE", "Checking for updates...");
17+
18+
let status = self_update::backends::github::Update::configure()

0 commit comments

Comments
 (0)