SCANPY-166 Ensure the JRE and scanner engine JAR is downloaded properly when the US region is set (#187)

Author: Seppli11

src/pysonar_scanner/api.py

@@ -41,6 +41,11 @@
 GLOBAL_SONARCLOUD_URL = "https://sonarcloud.io"
 US_SONARCLOUD_URL = "https://sonarqube.us"
 
+UNAUTHORIZED_STATUS_CODES = (401, 403)
+
+ACCEPT_JSON = {"Accept": "application/json"}
+ACCEPT_OCTET_STREAM = {"Accept": "application/octet-stream"}
+
 
 @dataclass(frozen=True)
 class SQVersion:
@@ -92,7 +97,7 @@ def __post_init__(self):
 
 @dataclass(frozen=True)
 class JRE:
-    id: str
+    id: Optional[str]
     filename: str
     sha256: str
     java_path: str
@@ -103,7 +108,7 @@ class JRE:
     @staticmethod
     def from_dict(dict: dict) -> "JRE":
         return JRE(
-            id=dict["id"],
+            id=dict.get("id", None),
             filename=dict["filename"],
             sha256=dict["sha256"],
             java_path=dict["javaPath"],
@@ -182,6 +187,7 @@ def __call__(self, r):
 class EngineInfo:
     filename: str
     sha256: str
+    download_url: Optional[str] = None
 
 
 class SonarQubeApi:
@@ -193,7 +199,7 @@ def __raise_exception(self, exception: Exception) -> NoReturn:
         if (
             isinstance(exception, requests.RequestException)
             and exception.response is not None
-            and exception.response.status_code == 401
+            and exception.response.status_code in UNAUTHORIZED_STATUS_CODES
         ):
             raise SonarQubeApiUnauthroizedException.create_default(self.base_urls.base_url) from exception
         else:
@@ -215,14 +221,14 @@ def get_analysis_version(self) -> SQVersion:
 
     def get_analysis_engine(self) -> EngineInfo:
         try:
-            res = requests.get(
-                f"{self.base_urls.api_base_url}/analysis/engine", headers={"Accept": "application/json"}, auth=self.auth
-            )
+            res = requests.get(f"{self.base_urls.api_base_url}/analysis/engine", headers=ACCEPT_JSON, auth=self.auth)
             res.raise_for_status()
             json = res.json()
             if "filename" not in json or "sha256" not in json:
                 raise SonarQubeApiException("Invalid response from the server")
-            return EngineInfo(filename=json["filename"], sha256=json["sha256"])
+            return EngineInfo(
+                filename=json["filename"], sha256=json["sha256"], download_url=json.get("downloadUrl", None)
+            )
         except requests.RequestException as e:
             self.__raise_exception(e)
 
@@ -234,7 +240,7 @@ def download_analysis_engine(self, handle: typing.BinaryIO) -> None:
         try:
             res = requests.get(
                 f"{self.base_urls.api_base_url}/analysis/engine",
-                headers={"Accept": "application/octet-stream"},
+                headers=ACCEPT_OCTET_STREAM,
                 auth=self.auth,
             )
             self.__download_file(res, handle)
@@ -247,7 +253,7 @@ def get_analysis_jres(self, os: OsStr, arch: ArchStr) -> list[JRE]:
             res = requests.get(
                 f"{self.base_urls.api_base_url}/analysis/jres",
                 auth=self.auth,
-                headers={"Accept": "application/json"},
+                headers=ACCEPT_JSON,
                 params=params,
             )
             res.raise_for_status()
@@ -265,13 +271,27 @@ def download_analysis_jre(self, id: str, handle: typing.BinaryIO) -> None:
         try:
             res = requests.get(
                 f"{self.base_urls.api_base_url}/analysis/jres/{id}",
-                headers={"Accept": "application/octet-stream"},
+                headers=ACCEPT_OCTET_STREAM,
                 auth=self.auth,
             )
             self.__download_file(res, handle)
         except requests.RequestException as e:
             self.__raise_exception(e)
 
+    def download_file_from_url(self, url: str, handle: typing.BinaryIO) -> None:
+        """
+        This method can raise a SonarQubeApiException if the server doesn't respond successfully.
+        Alternative, if the file IO fails, an IOError or OSError can be raised.
+        """
+        try:
+            res = requests.get(
+                url,
+                headers=ACCEPT_OCTET_STREAM,
+            )
+            self.__download_file(res, handle)
+        except requests.RequestException as e:
+            self.__raise_exception(e)
+
     def __download_file(self, res: requests.Response, handle: typing.BinaryIO) -> None:
         res.raise_for_status()
         for chunk in res.iter_content(chunk_size=128):

src/pysonar_scanner/jre.py

@@ -102,7 +102,14 @@ def __download_jre(self, jre: JRE) -> Optional[pathlib.Path]:
         cache_file.filepath.unlink(missing_ok=True)
 
         with cache_file.open(mode="wb") as f:
-            self.api.download_analysis_jre(jre.id, f)
+            if jre.download_url is not None:
+                self.api.download_file_from_url(jre.download_url, f)
+            elif jre.id is not None:
+                self.api.download_analysis_jre(jre.id, f)
+            else:
+                raise JreProvisioningException(
+                    "Failed to download the JRE using SonarQube. If this problem persists, you can use the option --sonar-scanner-java-exe-path to use your own local JRE."
+                )
 
         return cache_file.filepath if cache_file.is_valid() else None
 

src/pysonar_scanner/scannerengine.py

@@ -25,7 +25,7 @@
 from threading import Thread
 from typing import IO, Callable, Optional
 
-from pysonar_scanner.api import SonarQubeApi
+from pysonar_scanner.api import EngineInfo, SonarQubeApi
 from pysonar_scanner.cache import Cache, CacheFile
 from pysonar_scanner.exceptions import ChecksumException
 from pysonar_scanner.jre import JREResolvedPath
@@ -127,12 +127,15 @@ def __download_and_verify(self) -> Optional[CacheFile]:
         cache_file = self.cache.get_file(engine_info.filename, engine_info.sha256)
         if not cache_file.is_valid():
             logging.debug("No valid cached analysis engine jar was found")
-            self.__download_scanner_engine(cache_file)
+            self.__download_scanner_engine(cache_file, engine_info)
         return cache_file if cache_file.is_valid() else None
 
-    def __download_scanner_engine(self, cache_file: CacheFile) -> None:
+    def __download_scanner_engine(self, cache_file: CacheFile, engine_info: EngineInfo) -> None:
         with cache_file.open(mode="wb") as f:
-            self.api.download_analysis_engine(f)
+            if engine_info.download_url is not None:
+                self.api.download_file_from_url(engine_info.download_url, f)
+            else:
+                self.api.download_analysis_engine(f)
 
 
 class ScannerEngine:

tests/its/test_minimal.py

@@ -28,7 +28,7 @@
 
 
 def test_minimal_project(sonarqube_client: SonarQubeClient, cli: CliClient):
-    process = cli.run_analysis(sources_dir="minimal")
+    process = cli.run_analysis(sources_dir="minimal", params=["--verbose"])
     assert process.returncode == 0, str(process.stdout)
 
     data = sonarqube_client.get_project_issues("minimal")

tests/unit/sq_api_utils.py

@@ -52,14 +52,20 @@ def mock_analysis_version(self, version: str = "", status: int = 200) -> respons
         return self.rsps.get(url=f"{self.api_url}/analysis/version", body=version, status=status)
 
     def mock_analysis_engine(
-        self, filename: Optional[str] = None, sha256: Optional[str] = None, status: int = 200
+        self,
+        filename: Optional[str] = None,
+        sha256: Optional[str] = None,
+        download_url: Optional[str] = None,
+        status: int = 200,
     ) -> responses.BaseResponse:
         def prepare_json_obj() -> dict:
             json_response = {}
             if filename:
                 json_response["filename"] = filename
             if sha256:
                 json_response["sha256"] = sha256
+            if download_url:
+                json_response["downloadUrl"] = download_url
             return json_response
 
         return self.rsps.get(
@@ -105,6 +111,22 @@ def mock_analysis_jre_download(
             match=[matchers.header_matcher({"Accept": "application/octet-stream"})],
         )
 
+    def mock_download_url(
+        self,
+        url: str,
+        body: bytes = b"",
+        status: int = 200,
+        redirect_url: Optional[str] = None,
+    ) -> responses.BaseResponse:
+
+        return self.rsps.get(
+            url=url,
+            body=body,
+            headers={"Location": redirect_url} if redirect_url else None,
+            status=status,
+            match=[matchers.header_matcher({"Accept": "application/octet-stream"})],
+        )
+
     def mock_server_version(self, version: str = "", status: int = 200) -> responses.BaseResponse:
         return self.rsps.get(url=f"{self.base_url}/api/server/version", body=version, status=status)
 

tests/unit/test_api.py

@@ -300,6 +300,17 @@ def test_get_analysis_engine(self):
             mocker.mock_analysis_engine(filename=engine_info.filename, sha256=engine_info.sha256)
             self.assertEqual(self.sq.get_analysis_engine(), engine_info)
 
+        with self.subTest("get_analysis_engine with downloadUrl"), sq_api_mocker() as mocker:
+            engine_info = EngineInfo(
+                filename="sonar-scanner-engine-shaded-8.9.0.43852-all.jar",
+                sha256="1234567890",
+                download_url="https://example.com",
+            )
+            mocker.mock_analysis_engine(
+                filename=engine_info.filename, sha256=engine_info.sha256, download_url="https://example.com"
+            )
+            self.assertEqual(self.sq.get_analysis_engine(), engine_info)
+
         with (
             self.subTest("get_analysis_engine returns error"),
             sq_api_mocker() as mocker,
@@ -362,6 +373,15 @@ def test_get_analysis_jres(self):
                 arch="x64",
                 download_url=None,
             ),
+            JRE(
+                id=None,
+                filename="jre3.tar.gz",
+                sha256="dummysha256value3",
+                java_path="/path/to/jre1/bin/java",
+                os="linux",
+                arch="x64",
+                download_url="https://example.com/jre3.tar.gz",
+            ),
         ]
 
         with self.subTest("get_analysis_jres works (linux)"), sq_api_mocker() as mocker:
@@ -431,6 +451,33 @@ def test_download_analysis_jre(self):
             # since the api is not mocked, requests will throw an exception
             self.sq.download_analysis_jre(jre_id, io.BytesIO())
 
+    def test_download_file_from_url(self):
+        jre_url = "https://sonarcloud.io/jres/OpenJDK17U-jre_x64_alpine-linux_hotspot_17.0.11_9.tar.gz"
+        jre_file_content = b"fake_jre_binary"
+        with self.subTest("download_jre_from_url without auth works"), sq_api_mocker() as mocker:
+            mocker.mock_download_url(url=jre_url, body=jre_file_content)
+            fake_file = io.BytesIO()
+
+            self.sq.download_file_from_url(jre_url, fake_file)
+
+            self.assertEqual(fake_file.getvalue(), jre_file_content)
+
+        with (
+            self.subTest("download_jre_from_url returns 404"),
+            sq_api_mocker() as mocker,
+            self.assertRaises(SonarQubeApiException),
+        ):
+            mocker.mock_download_url(url=jre_url, status=404)
+            self.sq.download_file_from_url(jre_url, io.BytesIO())
+
+        with (
+            self.subTest("download_jre_from_url: requests throws exception"),
+            sq_api_mocker() as mocker,
+            self.assertRaises(SonarQubeApiException),
+        ):
+            # since the api is not mocked, requests will throw an exception
+            self.sq.download_file_from_url(jre_url, io.BytesIO())
+
     def test_to_api_configuration(self):
         with self.subTest("Missing keys"):
             expected = {

tests/unit/test_jre.py

@@ -20,6 +20,7 @@
 import io
 import pathlib
 import tarfile
+from typing import cast
 from unittest.mock import Mock, patch
 from typing_extensions import TypedDict
 import unittest
@@ -32,7 +33,12 @@
     SONAR_SCANNER_OS,
     SONAR_SCANNER_SKIP_JRE_PROVISIONING,
 )
-from pysonar_scanner.exceptions import ChecksumException, NoJreAvailableException, UnsupportedArchiveFormat
+from pysonar_scanner.exceptions import (
+    ChecksumException,
+    JreProvisioningException,
+    NoJreAvailableException,
+    UnsupportedArchiveFormat,
+)
 from pysonar_scanner.jre import JREProvisioner, JREResolvedPath, JREResolver, JREResolverConfiguration
 from pysonar_scanner.utils import Os, Arch
 from tests.unit import sq_api_utils
@@ -80,6 +86,16 @@ def __setup_zip_file(self):
             download_url=None,
         )
 
+        self.zip_jre_with_download_url = JRE(
+            id="zip_jre",
+            filename=self.zip_name,
+            sha256=self.zip_checksum,
+            java_path="java",
+            os=Os.LINUX.value,
+            arch=Arch.AARCH64.value,
+            download_url="https://scanner.sonarqube.us/jres/OpenJDK17U-jre_x64_alpine-linux_hotspot_17.0.11_9.tar.gz",
+        )
+
     def __setup_tar_file(self):
         buffer = io.BytesIO()
         with tarfile.open(fileobj=buffer, mode="w:gz") as tar_file:
@@ -154,6 +170,41 @@ class JRETestCase(TypedDict):
                 self.assertTrue((unziped_dir / "readme.md").exists())
                 self.assertEqual((unziped_dir / "readme.md").read_bytes(), b"hello world")
 
+    def test_download_jre_with_download_url(self, get_os_mock, get_arch_mock):
+        jre = self.zip_jre_with_download_url
+        with sq_api_utils.sq_api_mocker() as mocker:
+            mocker.mock_analysis_jres(body=[sq_api_utils.jre_to_dict(jre)])
+            mocker.mock_download_url(url=cast(str, jre.download_url), body=self.zip_bytes, status=200)
+
+            provisioner = JREProvisioner(self.api, self.cache, utils.get_os().value, utils.get_arch().value)
+            jre_path = provisioner.provision()
+
+            cache_file = self.cache.get_file(jre.filename, self.zip_checksum)
+            self.assertTrue(cache_file.is_valid())
+
+            unziped_dir = self.cache.get_file_path("jre.zip_unzip")
+            self.assertEqual(jre_path, JREResolvedPath(unziped_dir / "java"))
+
+            self.assertTrue(unziped_dir.exists())
+            self.assertTrue((unziped_dir / "readme.md").exists())
+            self.assertEqual((unziped_dir / "readme.md").read_bytes(), b"hello world")
+
+    def test_downloaind_jre_with_neither_download_url_nor_id(self, *args):
+        jre = JRE(
+            id=None,
+            filename="jre.zip",
+            sha256=self.zip_checksum,
+            java_path="java",
+            os=Os.LINUX.value,
+            arch=Arch.AARCH64.value,
+            download_url=None,
+        )
+        with self.assertRaises(JreProvisioningException), sq_api_utils.sq_api_mocker() as mocker:
+            mocker.mock_analysis_jres(body=[sq_api_utils.jre_to_dict(jre)])
+
+            provisioner = JREProvisioner(self.api, self.cache, utils.get_os().value, utils.get_arch().value)
+            provisioner.provision()
+
     def test_invalid_checksum(self, *args):
         with self.assertRaises(ChecksumException), sq_api_utils.sq_api_mocker() as mocker:
             jre_dict = sq_api_utils.jre_to_dict(self.zip_jre)

tests/unit/test_scannerengine.py

@@ -183,6 +183,18 @@ def test_happy_path(self):
             self.assertTrue(self.test_file_path.exists())
             self.assertEqual(self.test_file_path.read_bytes(), self.test_file_content)
 
+    def test_happy_path_with_download_url(self):
+        with sq_api_utils.sq_api_mocker() as mocker:
+            mocker.mock_analysis_engine(
+                filename="scanner-engine.jar", sha256=self.test_file_checksum, download_url="http://example.com"
+            )
+            mocker.mock_download_url(url="http://example.com", body=self.test_file_content)
+
+            ScannerEngineProvisioner(self.api, self.cache).provision()
+
+            self.assertTrue(self.test_file_path.exists())
+            self.assertEqual(self.test_file_path.read_bytes(), self.test_file_content)
+
     def test_scanner_engine_is_cached(self):
         with sq_api_utils.sq_api_mocker(assert_all_requests_are_fired=False) as mocker:
             engine_info_rsps = mocker.mock_analysis_engine(