Redo localization for onboarding

.dprint.jsonc

@@ -53,7 +53,8 @@
         // Needs to be LF to have a working shebang.
         "packages/typescript/bin/tsc",
         "tsc/internal/bundled/source/**",
-        "tsc/internal/locale/lcl/**",
+        "tsc/internal/diagnostics/diagnosticMessages.generated.json",
+        "tsc/internal/diagnostics/loc/*.generated.json",
         "packages/typescript/dist/**",
         "packages/typescript/tsconfig.tsbuildinfo",
         "packages/vscode-typescript/dist/**",

Herebyfile.mjs

@@ -492,11 +492,25 @@ export const generateTristate = goGenerateTask("generate:tristate", [
 export const generateDiagnostics = goGenerateTask("generate:diagnostics", [
     {
         file: "tsc/internal/diagnostics/diagnostics.go",
-        inputs: ["generate.go", "diagnosticMessages.json", "extraDiagnosticMessages.json", "../{collections,json}/*.go", "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl"],
+        inputs: ["generate.go", "diagnosticMessages.json", "../../../tools/LocProject.json", "../{collections,json}/*.go", "loc/*.generated.json"],
         exclude: ["**/*_test.go"],
-        outputs: ["diagnostics_generated.go", "loc_generated.go", "loc/*.json.gz"],
+        outputs: ["diagnostics_generated.go", "diagnosticMessages.generated.json", "loc_generated.go", "loc/*.json.gz"],
         commands: [
-            ["go", "run", "generate.go", "-diagnostics", "diagnostics_generated.go", "-loc", "loc_generated.go", "-locdir", "loc"],
+            [
+                "go",
+                "run",
+                "generate.go",
+                "-diagnostics",
+                "diagnostics_generated.go",
+                "-loc",
+                "loc_generated.go",
+                "-locdir",
+                "loc",
+                "-locproject",
+                "../../../tools/LocProject.json",
+                "-locsource",
+                "diagnosticMessages.generated.json",
+            ],
             ["dprint", "fmt", "diagnostics_generated.go", "loc_generated.go"],
         ],
     },

tools/LocProject.json

@@ -0,0 +1,14 @@
+{
+    "Projects": [
+        {
+            "LocItems": [
+                {
+                    "SourceFile": "tsc\\internal\\diagnostics\\diagnosticMessages.generated.json",
+                    "Languages": "zh-CN;zh-TW;cs-CZ;de-DE;es-ES;fr-FR;it-IT;ja-JP;ko-KR;pl-PL;pt-BR;ru-RU;tr-TR",
+                    "CopyOption": "UsePlaceholders",
+                    "OutputPath": "tsc\\internal\\diagnostics\\loc\\{Lang}.generated.json"
+                }
+            ]
+        }
+    ]
+}

tools/pipelines/localization-update.yml

@@ -0,0 +1,198 @@
+trigger: none
+pr: none
+
+# OneLocBuild materializes packaged LCL handbacks under the artifact directory.
+# Its manifests map those files to their native repository paths. After applying
+# them, the diagnostics generator produces deterministic runtime gzip files.
+# The resulting changes update a stable GitHub branch and pull request.
+schedules:
+  - cron: '0 7 * * *'
+    displayName: Daily localization update
+    always: true
+    branches:
+      include:
+        - main
+
+variables:
+  - name: TYPESCRIPT_AUTOMATION_GITHUB_APP_CLIENT_ID
+    value: Iv23li4GolzJSEp1mzHI
+  - name: TYPESCRIPT_AUTOMATION_GITHUB_APP_KEY_ID
+    value: https://jststeam-passwords.vault.azure.net/keys/typescript-automation
+
+jobs:
+  - job: Generate
+    displayName: Generate localization update
+    pool:
+      vmImage: windows-latest
+
+    steps:
+      - checkout: self
+        clean: true
+        fetchDepth: 1
+        fetchTags: false
+        submodules: false
+
+      - template: /tools/pipelines/steps/setup-node-npm-ci.yml@self
+
+      - pwsh: |
+          $ErrorActionPreference = "Stop"
+          $match = Select-String -Path tsc/go.mod -Pattern '^go ([0-9]+\.[0-9]+)$'
+          if (!$match) {
+              throw "Unable to determine the Go version from tsc/go.mod"
+          }
+          Write-Host "##vso[task.setvariable variable=GoVersion]$($match.Matches[0].Groups[1].Value)"
+        displayName: Determine Go version
+
+      - task: GoTool@0
+        displayName: Install Go
+        inputs:
+          version: $(GoVersion)
+          goDownloadUrl: https://aka.ms/golang/release/latest
+
+      - task: AzureCLI@2
+        displayName: Create OneLocBuild token
+        inputs:
+          azureSubscription: devdiv-onelocbuild-ceapex
+          scriptType: pscore
+          scriptLocation: inlineScript
+          inlineScript: |
+            $ErrorActionPreference = "Stop"
+            $PSNativeCommandUseErrorActionPreference = $true
+            $token = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
+            Write-Host "##vso[task.setvariable variable=OneLocBuildToken;issecret=true]$token"
+
+      - task: OneLocBuild@3
+        displayName: Generate localized files
+        env:
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+        inputs:
+          locProj: tools/LocProject.json
+          outDir: $(Build.ArtifactStagingDirectory)
+          LclSource: lclFilesfromPackage
+          LclPackageId: $(OneLocLclPackageId)
+          packageSourceAuth: patAuth
+          patVariable: $(OneLocBuildToken)
+          isCreatePrSelected: false
+
+      - pwsh: |
+          $ErrorActionPreference = "Stop"
+          # OneLoc leaves native files in the artifact directory. Its manifests
+          # pair each generated path with the repository path selected by the
+          # LocProject CopyOption and OutputPath.
+          $locRoot = Join-Path "$(Build.ArtifactStagingDirectory)" "loc"
+          $generatedFiles = @(Get-Content -LiteralPath (Join-Path $locRoot "XLocFileList.txt"))
+          $destinationFiles = @(Get-Content -LiteralPath (Join-Path $locRoot "AllFileList.txt"))
+          if ($generatedFiles.Count -eq 0) {
+              throw "OneLocBuild did not produce any localized files"
+          }
+          if ($generatedFiles.Count -ne $destinationFiles.Count) {
+              throw "OneLocBuild produced mismatched source and destination manifests"
+          }
+
+          $sourceRoot = [IO.Path]::GetFullPath("$(Build.SourcesDirectory)") + [IO.Path]::DirectorySeparatorChar
+          $manifestDestinations = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
+          for ($i = 0; $i -lt $generatedFiles.Count; $i++) {
+              $source = [IO.Path]::GetFullPath($generatedFiles[$i])
+              $destination = [IO.Path]::GetFullPath($destinationFiles[$i])
+              if (!$destination.StartsWith($sourceRoot, [StringComparison]::OrdinalIgnoreCase)) {
+                  throw "Refusing to write localized file outside the checkout: $destination"
+              }
+              if (!(Test-Path -LiteralPath $source -PathType Leaf)) {
+                  throw "OneLocBuild did not produce $source"
+              }
+              $null = $manifestDestinations.Add($destination)
+          }
+
+          $handbackDirectory = Join-Path $sourceRoot "tsc/internal/diagnostics/loc"
+          Get-ChildItem -LiteralPath $handbackDirectory -Filter "*.generated.json" -File |
+              Where-Object { !$manifestDestinations.Contains($_.FullName) } |
+              Remove-Item -Force
+
+          for ($i = 0; $i -lt $generatedFiles.Count; $i++) {
+              $source = [IO.Path]::GetFullPath($generatedFiles[$i])
+              $destination = [IO.Path]::GetFullPath($destinationFiles[$i])
+              New-Item -Path (Split-Path -Parent $destination) -ItemType Directory -Force | Out-Null
+              Copy-Item -LiteralPath $source -Destination $destination -Force
+          }
+        displayName: Apply localized files
+
+      - pwsh: |
+          $ErrorActionPreference = "Stop"
+          $PSNativeCommandUseErrorActionPreference = $true
+          npx hereby generate:diagnostics
+        displayName: Generate localization artifacts
+
+      - task: CopyFiles@2
+        displayName: Add localization project to handoff
+        inputs:
+          SourceFolder: $(Build.SourcesDirectory)/tools
+          Contents: LocProject.json
+          TargetFolder: $(Build.ArtifactStagingDirectory)/loc
+
+      - task: PublishPipelineArtifact@1
+        displayName: Publish localization handoff
+        inputs:
+          targetPath: $(Build.ArtifactStagingDirectory)/loc
+          artifact: Loc
+
+      - template: /tools/pipelines/steps/create-github-app-token.yml
+        parameters:
+          azureSubscription: TypeScript-DevDiv-KeyVault
+          repositories: TypeScript
+          permissions: contents:write,pull_requests:write
+          insertSteps:
+            - pwsh: |
+                $ErrorActionPreference = "Stop"
+                $PSNativeCommandUseErrorActionPreference = $true
+
+                git add --all -- tsc/internal/diagnostics/loc tsc/internal/diagnostics/loc_generated.go
+                $changes = @(git diff --cached --name-only)
+                if ($changes.Count -eq 0) {
+                    Write-Host "Localization files are already up to date."
+                    exit 0
+                }
+
+                $branch = "localization-update"
+                git switch -c $branch
+                git config user.email "290192711+typescript-automation[bot]@users.noreply.github.com"
+                git config user.name "typescript-automation[bot]"
+                $hooksPath = Join-Path $env:AGENT_TEMPDIRECTORY "empty-git-hooks"
+                New-Item -Path $hooksPath -ItemType Directory -Force | Out-Null
+                git config core.hooksPath $hooksPath
+                git commit -m "Update localization files"
+
+                $basicAuth = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes("x-access-token:$env:GH_TOKEN"))
+                Write-Host "##vso[task.setsecret]$basicAuth"
+                git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic $basicAuth"
+                git remote set-url origin https://github.com/microsoft/TypeScript.git
+
+                $remoteBranch = git ls-remote --heads origin $branch
+                $branchExists = [bool] $remoteBranch
+                if ($branchExists) {
+                    git fetch origin "refs/heads/${branch}:refs/remotes/origin/${branch}"
+                }
+
+                $shouldPush = $true
+                if ($branchExists) {
+                    $localTree = git rev-parse 'HEAD^{tree}'
+                    $remoteTree = git rev-parse "origin/${branch}^{tree}"
+                    if ($localTree -eq $remoteTree) {
+                        Write-Host "Remote branch already contains the same files."
+                        $shouldPush = $false
+                    }
+                }
+
+                if ($shouldPush) {
+                    git push --force-with-lease --set-upstream origin $branch
+                }
+
+                $existingPr = gh pr list --repo microsoft/TypeScript --base main --head $branch --state open --json url --jq '.[0].url // empty'
+                if ($existingPr) {
+                    Write-Host "Updated existing pull request: $existingPr"
+                }
+                else {
+                    gh pr create --repo microsoft/TypeScript --base main --head $branch --title "Update localization files" --body "Updates localization handbacks and their generated runtime artifacts."
+                }
+              displayName: Create or update localization pull request
+              env:
+                GH_TOKEN: $(GH_TOKEN)

tools/pipelines/steps/create-github-app-token.yml

@@ -30,10 +30,10 @@ steps:
     displayName: Create GitHub App token
     inputs:
       azureSubscription: ${{ parameters.azureSubscription }}
-      scriptType: bash
+      scriptType: pscore
       scriptLocation: inlineScript
       inlineScript: |
-        cat << 'GITHUB_APP_TOKEN_CLI_EOF' > /tmp/create-github-app-token.cjs
+        @'
         "use strict";
 
         // src/api.ts
@@ -589,8 +589,8 @@ steps:
           reportError(error);
           process.exitCode = 1;
         });
-        GITHUB_APP_TOKEN_CLI_EOF
-        node /tmp/create-github-app-token.cjs
+        '@ | Set-Content -LiteralPath "$(Agent.TempDirectory)/create-github-app-token.cjs" -Encoding utf8NoBOM
+        node "$(Agent.TempDirectory)/create-github-app-token.cjs"
     env:
       APP_CLIENT_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_CLIENT_ID)
       KEY_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_KEY_ID)
@@ -603,12 +603,13 @@ steps:
   - ${{ each step in parameters.insertSteps }}:
       - ${{ step }}
 
-  - task: Bash@3
+  - task: PowerShell@2
     displayName: Revoke GitHub App token
     condition: always()
     continueOnError: true
     inputs:
       targetType: inline
-      script: node /tmp/create-github-app-token.cjs
+      pwsh: true
+      script: node "$(Agent.TempDirectory)/create-github-app-token.cjs"
     env:
       REVOKE_TOKEN: $(${{ parameters.tokenVariable }})

tools/scripts/gen/generatedFile.test.mts

@@ -649,25 +649,42 @@ test("diagnostic generation tracks Go and locale outputs and supports force", as
     const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_diagnostics-probe-"));
     context.after(() => fs.rmSync(directory, { recursive: true, force: true }));
     const output = path.join(directory, "diagnostics_generated.go");
+    const source = path.join(directory, "diagnosticMessages.generated.json");
     const localized = path.join(directory, "loc_generated.go");
+    const locDirectory = path.join(directory, "loc");
+    fs.mkdirSync(locDirectory);
+    const handbacks = fs.globSync("loc/*.generated.json", { cwd: path.join(root, "tsc/internal/diagnostics") });
+    for (const handback of handbacks) {
+        fs.copyFileSync(path.join(root, "tsc/internal/diagnostics", handback), path.join(locDirectory, path.basename(handback)));
+    }
+    const project = path.join(directory, "tools/LocProject.json");
+    fs.mkdirSync(path.dirname(project));
+    const projectData = JSON.parse(fs.readFileSync(path.join(root, "tools/LocProject.json"), "utf8"));
+    projectData.Projects[0].LocItems = [projectData.Projects[0].LocItems[0]];
+    projectData.Projects[0].LocItems[0].SourceFile = "diagnosticMessages.generated.json";
+    fs.writeFileSync(project, JSON.stringify(projectData));
     const options = {
         cwd: path.join(root, "tsc/internal/diagnostics"),
-        inputs: ["generate.go", "diagnosticMessages.json", "extraDiagnosticMessages.json", "../{collections,json}/*.go", "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl"],
+        inputs: ["generate.go", "diagnosticMessages.json", project, "../{collections,json}/*.go", path.join(locDirectory, "*.generated.json")],
         exclude: ["**/*_test.go"],
-        outputs: [output, localized, path.join(directory, "loc/*.json.gz")],
+        outputs: [output, source, localized, path.join(locDirectory, "*.json.gz")],
         commands: [
-            ["go", "run", "generate.go", "-diagnostics", output, "-loc", localized, "-locdir", path.join(directory, "loc")],
+            ["go", "run", "generate.go", "-diagnostics", output, "-loc", localized, "-locdir", locDirectory, "-locproject", project, "-locsource", source],
             ["dprint", "fmt", output, localized],
         ],
     };
     const generate = (force = false) => cache({ ...options, force });
     await generate();
     const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file));
     assert.ok(locales.length > 0);
-    const files = [output, localized, ...locales];
+    const files = [output, source, localized, ...locales];
     const timestamps = files.map(file => fs.statSync(file).mtimeMs);
     assert.equal(await generate(), true);
     assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps);
+    fs.appendFileSync(project, "\n");
+    assert.equal(await generate(), false);
+    fs.appendFileSync(path.join(locDirectory, path.basename(handbacks[0])), "\n");
+    assert.equal(await generate(), false);
     const archive = fs.readFileSync(locales[0]);
     fs.rmSync(locales[0]);
     assert.equal(await generate(), false);
@@ -679,7 +696,7 @@ test("diagnostic generation tracks Go and locale outputs and supports force", as
     fs.writeFileSync(unexpected, "unexpected");
     assert.equal(await generate(), false);
     assert.equal(fs.existsSync(unexpected), false);
-    fs.rmSync(path.join(directory, "loc"), { recursive: true });
+    for (const locale of locales) fs.rmSync(locale);
     assert.equal(await generate(), false);
     assert.deepEqual(fs.readFileSync(locales[0]), archive);
     fs.rmSync(localized);