[Cosmos] Backport PPCB fixes to 4.76.1-hotfix

NuGet.Config

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <!-- Do not add any additional feeds. If new packages are needed they need to
+    come from the azure-sdk-for-net DevOps feed which has an upstream set to nuget.org -->
+    <add key="azure-sdk-for-net" value="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

eng/common/testproxy/test-proxy-tool.yml

@@ -5,6 +5,7 @@ parameters:
   targetVersion: ''
   templateRoot: '$(Build.SourcesDirectory)'
   condition: true
+  proxyUrl: 'http://localhost:5000'
 
 steps:
   - pwsh: |
@@ -67,14 +68,14 @@ steps:
     - pwsh: |
         $invocation = @"
         Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe
-          -ArgumentList `"start -u --storage-location ${{ parameters.rootFolder }}`"
+          -ArgumentList `"start -u --storage-location ${{ parameters.rootFolder }} -- --urls ${{ parameters.proxyUrl }}`"
           -NoNewWindow -PassThru -RedirectStandardOutput ${{ parameters.rootFolder }}/test-proxy.log
           -RedirectStandardError ${{ parameters.rootFolder }}/test-proxy-error.log
         "@
         Write-Host $invocation
 
         $Process = Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe `
-          -ArgumentList "start -u --storage-location ${{ parameters.rootFolder }}" `
+          -ArgumentList "start -u --storage-location ${{ parameters.rootFolder }} -- --urls ${{ parameters.proxyUrl }}" `
           -NoNewWindow -PassThru -RedirectStandardOutput ${{ parameters.rootFolder }}/test-proxy.log `
           -RedirectStandardError ${{ parameters.rootFolder }}/test-proxy-error.log
 
@@ -87,7 +88,10 @@ steps:
 
     # nohup does NOT continue beyond the current session if you use it within powershell
     - bash: |
-        nohup $(Build.BinariesDirectory)/test-proxy/test-proxy 1>${{ parameters.rootFolder }}/test-proxy.log 2>${{ parameters.rootFolder }}/test-proxy-error.log &
+        if [[ "$(uname)" == "Darwin" ]]; then
+          export DOTNET_ROOT="$HOME/.dotnet"
+        fi
+        nohup $(Build.BinariesDirectory)/test-proxy/test-proxy start -u --storage-location ${{ parameters.rootFolder }} -- --urls "${{ parameters.proxyUrl }}" 1>${{ parameters.rootFolder }}/test-proxy.log 2>${{ parameters.rootFolder }}/test-proxy-error.log &
 
         echo $! > $(Build.SourcesDirectory)/test-proxy.pid
 
@@ -102,8 +106,8 @@ steps:
     - pwsh: |
         for ($i = 0; $i -lt 10; $i++) {
             try {
-                Write-Host "Invoke-WebRequest -Uri `"http://localhost:5000/Admin/IsAlive`" | Out-Null"
-                Invoke-WebRequest -Uri "http://localhost:5000/Admin/IsAlive" | Out-Null
+            Write-Host "Invoke-WebRequest -Uri `"${{ parameters.proxyUrl }}/Admin/IsAlive`" | Out-Null"
+            Invoke-WebRequest -Uri "${{ parameters.proxyUrl }}/Admin/IsAlive" | Out-Null
                 Write-Host "Successfully connected to the test proxy on port 5000."
                 exit 0
             } catch {

eng/pipelines/templates/jobs/ci.tests.yml

@@ -127,6 +127,9 @@ jobs:
 
       - ${{ parameters.PreTestSteps }}
 
+      # Authenticate with Azure Artifacts
+      - template: /eng/pipelines/templates/steps/maven-authenticate.yml
+
       - template: /eng/pipelines/templates/steps/run-and-validate-linting.yml
         parameters:
           JavaBuildVersion: $(JavaTestVersion)

eng/pipelines/templates/jobs/ci.yml

@@ -115,6 +115,9 @@ jobs:
           ServiceDirectory: ${{parameters.ServiceDirectory}}
           ExcludePaths: ${{parameters.ExcludePaths}}
 
+      # Authenticate with Azure Artifacts
+      - template: /eng/pipelines/templates/steps/maven-authenticate.yml
+
       - task: UsePythonVersion@0
         displayName: 'Use Python $(PythonVersion)'
         inputs:
@@ -191,6 +194,11 @@ jobs:
           parameters:
             PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
+      - task: PipAuthenticate@1
+        displayName: 'Pip Authenticate to Azure Artifacts'
+        inputs:
+          artifactFeeds: 'public/azure-sdk-for-python'
+
       - script: |
           python -m pip install markdown2==2.4.6 BeautifulSoup4==4.11.1
         displayName: 'pip install markdown2 and BeautifulSoup4'
@@ -402,6 +410,11 @@ jobs:
           version: 22.x
         displayName: Use Node.js 22.x
 
+      - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
+        parameters:
+          npmrcPath: $(Agent.TempDirectory)/analyze-job/.npmrc
+          registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/
+
       - task: PowerShell@2
         displayName: Verify Swagger and TypeSpec Code Generation
         inputs:
@@ -410,6 +423,11 @@ jobs:
           arguments: >
             -ServiceDirectories '$(PRServiceDirectories)'
             -RegenerationType 'All'
+        env:
+          npm_config_userconfig: $(Agent.TempDirectory)/analyze-job/.npmrc
+
+      # Authenticate with Azure Artifacts
+      - template: /eng/pipelines/templates/steps/maven-authenticate.yml
 
       - template: /eng/pipelines/templates/steps/run-and-validate-linting.yml
         parameters:

eng/pipelines/templates/jobs/live.tests.yml

@@ -121,6 +121,9 @@ jobs:
 
       - ${{ parameters.PreSteps }}
 
+      # Authenticate with Azure Artifacts
+      - template: /eng/pipelines/templates/steps/maven-authenticate.yml
+
       - template: /eng/pipelines/templates/steps/build-and-test.yml
         parameters:
           PreTestRunSteps: ${{ parameters.PreTestRunSteps }}

eng/pipelines/templates/steps/maven-authenticate.yml

@@ -0,0 +1,14 @@
+steps:
+  # Copy mirror settings to default Maven location so all requests go through CFS
+  - pwsh: |
+      $m2Dir = if ($env:USERPROFILE) { "$env:USERPROFILE\.m2" } else { "$HOME/.m2" }
+      New-Item -ItemType Directory -Force -Path $m2Dir | Out-Null
+      Copy-Item -Path "$(Build.SourcesDirectory)/eng/settings.xml" -Destination "$m2Dir/settings.xml"
+    displayName: 'Setup Maven mirror settings'
+
+  # Authenticate with Azure Artifacts feeds
+  # MavenAuthenticate adds <server> entries to ~/.m2/settings.xml matching mirror id 'azure-sdk-for-java'
+  - task: MavenAuthenticate@0
+    displayName: 'Maven Authenticate'
+    inputs:
+      artifactsFeeds: 'azure-sdk-for-java'

eng/pipelines/templates/variables/globals.yml

@@ -26,7 +26,7 @@ variables:
   # See https://github.com/actions/virtual-environments/issues/1499 for more info about the wagon options
   # If reports about Maven dependency downloads become more common investigate re-introducing "-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false", or other iterations of the configurations.
   WagonOptions: '-Dmaven.wagon.httpconnectionManager.ttlSeconds=60 -Dmaven.wagon.http.pool=false'
-  DefaultOptions: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --batch-mode --fail-at-end --settings eng/settings.xml $(WagonOptions)'
+  DefaultOptions: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --batch-mode --fail-at-end $(WagonOptions)'
   LoggingOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=$(MavenLogLevel) -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn'
   MemoryOptions: '-Xmx4096m'
   DefaultSkipOptions: '-Dgpg.skip -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -Dspotless.apply.skip=true -Dspotless.check.skip=true'

eng/settings.xml

@@ -1,4 +1,13 @@
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" 
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
-                      https://maven.apache.org/xsd/settings-1.0.0.xsd">
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+                                                https://maven.apache.org/xsd/settings-1.0.0.xsd">
+    <mirrors>
+        <mirror>
+            <id>azure-sdk-for-java</id>
+            <name>Azure Artifacts Maven Mirror</name>
+            <url>https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-java/maven/v1</url>
+            <mirrorOf>external:*</mirrorOf>
+        </mirror>
+    </mirrors>
 </settings>

eng/versioning/external_dependencies.txt

@@ -286,7 +286,6 @@ cosmos_org.apache.kafka:connect-runtime;3.6.0
 cosmos_org.testcontainers:testcontainers;1.19.5
 cosmos_org.testcontainers:kafka;1.19.5
 cosmos_org.sourcelab:kafka-connect-client;4.0.4
-cosmos_io.confluent:kafka-avro-serializer;7.6.0
 cosmos_org.apache.avro:avro;1.11.4
 # Maven Tools for Cosmos Kafka connector only
 

eng/versioning/version_client.txt

@@ -104,7 +104,7 @@ com.azure:azure-core-test;1.27.0-beta.13;1.27.0-beta.14
 com.azure:azure-core-tracing-opentelemetry;1.0.0-beta.61;1.0.0-beta.62
 com.azure:azure-core-tracing-opentelemetry-samples;1.0.0-beta.1;1.0.0-beta.1
 com.azure:azure-core-version-tests;1.0.0-beta.1;1.0.0-beta.1
-com.azure:azure-cosmos;4.75.0;4.76.0
+com.azure:azure-cosmos;4.75.0;4.76.1-hotfix
 com.azure:azure-cosmos-benchmark;4.0.1-beta.1;4.0.1-beta.1
 com.azure.cosmos.spark:azure-cosmos-spark_3;0.0.1-beta.1;0.0.1-beta.1
 com.azure.cosmos.spark:azure-cosmos-spark_3-5;0.0.1-beta.1;0.0.1-beta.1

sdk/cosmos/azure-cosmos-benchmark/pom.xml

@@ -52,7 +52,7 @@ Licensed under the MIT License.
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-cosmos</artifactId>
-      <version>4.76.0</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
+      <version>4.76.1-hotfix</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
     </dependency>
 
     <dependency>

sdk/cosmos/azure-cosmos-encryption/pom.xml

@@ -61,7 +61,7 @@ Licensed under the MIT License.
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-cosmos</artifactId>
-      <version>4.76.0</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
+      <version>4.76.1-hotfix</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
     </dependency>
 
     <dependency>

sdk/cosmos/azure-cosmos-kafka-connect/pom.xml

@@ -27,11 +27,6 @@ Licensed under the MIT License.
   </distributionManagement>
 
   <repositories>
-    <repository>
-      <id>confluent</id>
-      <name>Confluent</name>
-      <url>https://packages.confluent.io/maven/</url>
-    </repository>
     <repository>
       <id>maven-repo1</id>
       <name>Maven Repo1</name>
@@ -92,7 +87,7 @@ Licensed under the MIT License.
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-cosmos</artifactId>
-      <version>4.76.0</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
+      <version>4.76.1-hotfix</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
     </dependency>
 
     <!-- Added this provided dependency to include necessary annotations used by "reactor-core".
@@ -242,13 +237,6 @@ Licensed under the MIT License.
       <version>4.0.4</version> <!-- {x-version-update;cosmos_org.sourcelab:kafka-connect-client;external_dependency} -->
       <scope>test</scope>
     </dependency>
-    <!-- https://mvnrepository.com/artifact/io.confluent/kafka-avro-serializer -->
-    <dependency>
-      <groupId>io.confluent</groupId>
-      <artifactId>kafka-avro-serializer</artifactId>
-      <version>7.6.0</version><!-- {x-version-update;cosmos_io.confluent:kafka-avro-serializer;external_dependency} -->
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
@@ -419,7 +407,6 @@ Licensed under the MIT License.
                   <artifactSet>
                     <excludes>
                       <exclude>org.slf4j</exclude>
-                      <exclude>io.confluent:*</exclude>
                       <exclude>org.apache.kafka:*</exclude>
                     </excludes>
                   </artifactSet>

sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosSinkConnectorITest.java

@@ -12,7 +12,6 @@
 import com.azure.cosmos.kafka.connect.implementation.sink.IdStrategyType;
 import com.azure.cosmos.models.PartitionKey;
 import com.fasterxml.jackson.databind.JsonNode;
-import io.confluent.kafka.serializers.KafkaAvroSerializer;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerConfig;
@@ -286,8 +285,8 @@ public void postAvroMessage() throws InterruptedException {
             kafkaCosmosConnectContainer.registerConnector(connectorName, sinkConnectorConfig);
 
             Properties producerProperties = kafkaCosmosConnectContainer.getProducerProperties();
-            producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
-            producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
+            producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, TestAvroSerializer.class.getName());
+            producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, TestAvroSerializer.class.getName());
             KafkaProducer<GenericRecord, GenericRecord> kafkaProducer = new KafkaProducer<>(producerProperties);
 
             // first create few records in the topic
@@ -364,8 +363,8 @@ public void postAvroMessageWithTemplateIdStrategy() throws InterruptedException
             kafkaCosmosConnectContainer.registerConnector(connectorName, sinkConnectorConfig);
 
             Properties producerProperties = kafkaCosmosConnectContainer.getProducerProperties();
-            producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
-            producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
+            producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, TestAvroSerializer.class.getName());
+            producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, TestAvroSerializer.class.getName());
             KafkaProducer<GenericRecord, GenericRecord> kafkaProducer = new KafkaProducer<>(producerProperties);
 
             logger.info("Creating sink record...");
@@ -432,8 +431,8 @@ public void postAvroMessageWithJsonPathInProvidedInKeyStrategy() throws Interrup
             kafkaCosmosConnectContainer.registerConnector(connectorName, sinkConnectorConfig);
 
             Properties producerProperties = kafkaCosmosConnectContainer.getProducerProperties();
-            producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
-            producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
+            producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, TestAvroSerializer.class.getName());
+            producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, TestAvroSerializer.class.getName());
             KafkaProducer<GenericRecord, GenericRecord> kafkaProducer = new KafkaProducer<>(producerProperties);
 
             logger.info("Creating sink record...");