feat: Do not collect git metadata by default when organization-level git metadata settings are absent

.changeset/soft-eagles-protect.md

@@ -0,0 +1,5 @@
+---
+"braintrust": patch
+---
+
+feat: Do not collect git metadata by default when organization-level git metadata settings are absent

js/src/framework.ts

@@ -344,7 +344,7 @@ export interface Evaluator<
   baseExperimentId?: string;
 
   /**
-   * Optional settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings.
+   * Optional settings for collecting git metadata. By default, Braintrust collects the git metadata fields allowed by your organization's git metadata settings. If those settings are absent, git metadata is not collected unless this option is set.
    */
   gitMetadataSettings?: GitMetadataSettings;
 

js/src/gitutil.ts

@@ -149,12 +149,12 @@ function truncateToByteLimit(s: string, byteLimit: number = 65536): string {
 }
 
 export async function getRepoInfo(settings?: GitMetadataSettings) {
-  if (settings && settings.collect === "none") {
+  if (!settings || settings.collect === "none") {
     return undefined;
   }
 
   const repo = await repoInfo();
-  if (!repo || !settings || settings.collect === "all") {
+  if (!repo || settings.collect === "all") {
     return repo;
   }
 

js/src/logger.test.ts

@@ -21,6 +21,7 @@ import {
 } from "./logger";
 
 import { configureNode } from "./node/config";
+import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types";
 import { writeFile, unlink } from "node:fs/promises";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
@@ -484,6 +485,86 @@ function mockInitGitMetadata() {
   ).mockResolvedValue([]);
 }
 
+const initGitMetadataSettingsCases: Array<{
+  name: string;
+  orgSettings?: GitMetadataSettings;
+  initSettings?: GitMetadataSettings;
+  expected: GitMetadataSettings;
+}> = [
+  {
+    name: "uses organization settings by default",
+    orgSettings: { collect: "some", fields: ["commit", "branch"] },
+    expected: { collect: "some", fields: ["commit", "branch"] },
+  },
+  {
+    name: "does not collect by default when organization settings are absent",
+    expected: { collect: "none" },
+  },
+  {
+    name: "preserves explicit opt-in when organization settings are absent",
+    initSettings: { collect: "all" },
+    expected: { collect: "all" },
+  },
+  {
+    name: "intersects explicit settings with organization settings",
+    orgSettings: { collect: "some", fields: ["commit", "branch"] },
+    initSettings: { collect: "some", fields: ["branch", "git_diff"] },
+    expected: { collect: "some", fields: ["branch"] },
+  },
+  {
+    name: "lets organization settings constrain explicit all",
+    orgSettings: { collect: "some", fields: ["commit"] },
+    initSettings: { collect: "all" },
+    expected: { collect: "some", fields: ["commit"] },
+  },
+];
+
+test.each(initGitMetadataSettingsCases)(
+  "init applies git metadata settings: $name",
+  async ({ orgSettings, initSettings, expected }) => {
+    const state = await _exportsForTestingOnly.simulateLoginForTests();
+
+    try {
+      state.gitMetadataSettings = orgSettings;
+      vi.spyOn(state, "login").mockResolvedValue(state);
+      const getRepoInfo = vi
+        .spyOn(_exportsForTestingOnly.isomorph, "getRepoInfo")
+        .mockResolvedValue(undefined);
+      vi.spyOn(
+        _exportsForTestingOnly.isomorph,
+        "getPastNAncestors",
+      ).mockResolvedValue([]);
+      vi.spyOn(state.appConn(), "post_json").mockResolvedValue({
+        project: {
+          id: "00000000-0000-0000-0000-000000000001",
+          name: "test-project",
+        },
+        experiment: {
+          id: "00000000-0000-0000-0000-000000000003",
+          project_id: "00000000-0000-0000-0000-000000000001",
+          name: "test-experiment",
+          public: false,
+        },
+      });
+
+      const experiment = init({
+        project: "test-project",
+        experiment: "test-experiment",
+        gitMetadataSettings: initSettings,
+        setCurrent: false,
+        state,
+      });
+
+      await experiment.id;
+
+      expect(getRepoInfo).toHaveBeenCalledWith(expected);
+    } finally {
+      _exportsForTestingOnly.simulateLogoutForTests();
+      vi.restoreAllMocks();
+    }
+  },
+);
+
 test("init forwards dataset _internal_btql to experiment register", async () => {
   const state = await _exportsForTestingOnly.simulateLoginForTests();
 

js/src/logger.ts

@@ -3545,7 +3545,7 @@ type InitializedExperiment<IsOpen extends boolean | undefined> =
  * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login.
  * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
  * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
- * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings.
+ * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, Braintrust collects the git metadata fields allowed by your organization's git metadata settings. If those settings are absent, git metadata is not collected unless this option is set.
  * @param setCurrent If true (the default), set the global current-experiment to the newly-created one.
  * @param options.open If the experiment already exists, open it in read-only mode. Throws an error if the experiment does not already exist.
  * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified.
@@ -3698,17 +3698,13 @@ export function init<IsOpen extends boolean = false>(
         if (repoInfo) {
           return repoInfo;
         }
-        let mergedGitMetadataSettings = {
-          ...(state.gitMetadataSettings || {
-            collect: "all",
-          }),
-        };
-        if (gitMetadataSettings) {
-          mergedGitMetadataSettings = mergeGitMetadataSettings(
-            mergedGitMetadataSettings,
-            gitMetadataSettings,
-          );
-        }
+        const mergedGitMetadataSettings =
+          state.gitMetadataSettings == null
+            ? (gitMetadataSettings ?? { collect: "none" as const })
+            : mergeGitMetadataSettings(
+                state.gitMetadataSettings,
+                gitMetadataSettings ?? { collect: "all" as const },
+              );
         return await iso.getRepoInfo(mergedGitMetadataSettings);
       })();