SCANPY-148 Stream logs from the scanner to the user (#157)

Author: Seppli11

poetry.lock

@@ -18,21 +18,30 @@
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 
-from pysonar_scanner import cache
+from pysonar_scanner import app_logging, cache
 from pysonar_scanner import configuration
 from pysonar_scanner.api import get_base_urls, SonarQubeApi
 from pysonar_scanner.configuration import ConfigurationLoader
+from pysonar_scanner.configuration.properties import SONAR_VERBOSE
 from pysonar_scanner.scannerengine import ScannerEngine
 
 
 def scan():
+    app_logging.setup()
+
     config = ConfigurationLoader.load()
+    set_logging_options(config)
+
     cache_manager = cache.get_default()
     api = __build_api(config)
     scanner = ScannerEngine(api, cache_manager)
     return scanner.run(config)
 
 
+def set_logging_options(config):
+    app_logging.configure_logging_level(verbose=config.get(SONAR_VERBOSE, False))
+
+
 def __build_api(config: dict[str, any]) -> SonarQubeApi:
     token = configuration.get_token(config)
     base_urls = get_base_urls(config)

src/pysonar_scanner/__main__.py

@@ -0,0 +1,28 @@
+#
+# Sonar Scanner Python
+# Copyright (C) 2011-2024 SonarSource SA.
+# mailto:info AT sonarsource DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+#
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+import logging
+
+
+def setup() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+
+def configure_logging_level(verbose: bool) -> None:
+    logging.getLogger().setLevel(logging.DEBUG if verbose else logging.INFO)

src/pysonar_scanner/app_logging.py

@@ -17,9 +17,15 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
+from enum import Enum
 import json
+import logging
+from operator import le
 import pathlib
-from typing import Optional
+from threading import Thread
+from typing import IO, Callable, Optional
+
+from dataclasses import dataclass
 
 import pysonar_scanner.api as api
 
@@ -30,6 +36,80 @@
 from subprocess import Popen, PIPE
 
 
+@dataclass(frozen=True)
+class LogLine:
+    level: str
+    message: str
+    stacktrace: Optional[str] = None
+
+    def get_logging_level(self) -> int:
+        if self.level == "ERROR":
+            return logging.ERROR
+        if self.level == "WARN":
+            return logging.WARNING
+        if self.level == "INFO":
+            return logging.INFO
+        if self.level == "DEBUG":
+            return logging.DEBUG
+        if self.level == "TRACE":
+            return logging.DEBUG
+        return logging.INFO
+
+
+def parse_log_line(line: str) -> LogLine:
+    try:
+        line_json = json.loads(line)
+        level = line_json.get("level", "INFO")
+        message = line_json.get("message", line)
+        stacktrace = line_json.get("stacktrace")
+        return LogLine(level=level, message=message, stacktrace=stacktrace)
+    except json.JSONDecodeError:
+        return LogLine(level="INFO", message=line, stacktrace=None)
+
+
+def default_log_line_listener(log_line: LogLine):
+    logging.log(log_line.get_logging_level(), log_line.message)
+    if log_line.stacktrace is not None:
+        logging.log(log_line.get_logging_level(), log_line.stacktrace)
+
+
+class CmdExecutor:
+    def __init__(
+        self,
+        cmd: list[str],
+        properties_str: str,
+        log_line_listener: Callable[[LogLine], None] = default_log_line_listener,
+    ):
+        self.cmd = cmd
+        self.properties_str = properties_str
+        self.log_line_listener = log_line_listener
+
+    def execute(self):
+        process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        process.stdin.write(self.properties_str.encode())
+        process.stdin.close()
+
+        output_thread = Thread(target=self.__log_output, args=(process.stdout,))
+        error_thread = Thread(target=self.__log_output, args=(process.stderr,))
+
+        return self.__process_output(output_thread, error_thread, process)
+
+    def __log_output(self, stream: IO[bytes]):
+        for line in stream:
+            decoded_line = line.decode("utf-8").rstrip()
+            log_line = parse_log_line(decoded_line)
+            self.log_line_listener(log_line)
+
+    def __process_output(self, output_thread: Thread, error_thread: Thread, process: Popen) -> int:
+        output_thread.start()
+        error_thread.start()
+        process.wait()
+        output_thread.join()
+        error_thread.join()
+
+        return process.returncode
+
+
 class ScannerEngineProvisioner:
     def __init__(self, api: SonarQubeApi, cache: Cache):
         self.api = api
@@ -71,7 +151,8 @@ def run(self, config: dict[str, any]):
         jre_path = self.__resolve_jre(config)
         scanner_engine_path = self.__fetch_scanner_engine()
         cmd = self.__build_command(jre_path, scanner_engine_path)
-        return self.__execute_scanner_engine(config, cmd)
+        properties_str = self.__config_to_json(config)
+        return CmdExecutor(cmd, properties_str).execute()
 
     def __build_command(self, jre_path: JREResolvedPath, scanner_engine_path: pathlib.Path) -> list[str]:
         cmd = []
@@ -80,35 +161,10 @@ def __build_command(self, jre_path: JREResolvedPath, scanner_engine_path: pathli
         cmd.append(scanner_engine_path)
         return cmd
 
-    def __execute_scanner_engine(self, config: dict[str, any], cmd: list[str]) -> int:
-        popen = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
-        outs, _ = popen.communicate(self.__config_to_json(config).encode())
-        exitcode = popen.wait()  # 0 means success
-        self.__extract_errors_from_log(outs)
-        if exitcode != 0:
-            errors = self.__extract_errors_from_log(outs)
-            raise RuntimeError(f"Scan failed with exit code {exitcode}", errors)
-        return exitcode
-
     def __config_to_json(self, config: dict[str, any]) -> str:
         scanner_properties = [{"key": k, "value": v} for k, v in config.items()]
         return json.dumps({"scannerProperties": scanner_properties})
 
-    def __extract_errors_from_log(self, outs: str) -> list[str]:
-        try:
-            errors = []
-            for line in outs.decode("utf-8").split("\n"):
-                if line.strip() == "":
-                    continue
-                out_json = json.loads(line)
-                if out_json["level"] == "ERROR":
-                    errors.append(out_json["message"])
-                print(f"{out_json['level']}: {out_json['message']}")
-            return errors
-        except Exception as e:
-            print(e)
-            return []
-
     def __version_check(self):
         if self.api.is_sonar_qube_cloud():
             return

src/pysonar_scanner/scannerengine.py

@@ -17,52 +17,20 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
-import unittest
 from unittest import mock
 from unittest.mock import patch
 
-from pysonar_scanner.exceptions import MissingKeyException
-from pysonar_scanner.jre import JRE, JREResolvedPath
+from pyfakefs import fake_filesystem_unittest as pyfakefs
+
+from pysonar_scanner.configuration import ConfigurationLoader
+from pysonar_scanner.configuration.properties import SONAR_PROJECT_KEY, SONAR_TOKEN
 from pysonar_scanner.__main__ import scan
-from pysonar_scanner import __main__ as main
 from pysonar_scanner.scannerengine import ScannerEngine
 
 
-class TestMain(unittest.TestCase):
-
-    @patch("pysonar_scanner.scannerengine.Popen")
-    @patch.object(ScannerEngine, "_ScannerEngine__resolve_jre")
-    @patch.object(ScannerEngine, "_ScannerEngine__fetch_scanner_engine")
-    @patch(
-        "sys.argv",
-        ["pysonar-scanner", "-t", "myToken", "--sonar-project-key", "myProjectKey"],
-    )
-    def test_minimal_run_success(self, mock_fetch_scanner_engine, mock_resolve_jre, mock_popen):
-        mock_resolve_jre.return_value = JREResolvedPath("")
-        mock_fetch_scanner_engine.return_value = None
-        process_mock = mock.Mock()
-        attrs = {"communicate.return_value": ("output", "error"), "wait.return_value": 0}
-        process_mock.configure_mock(**attrs)
-        mock_popen.return_value = process_mock
-
+class TestMain(pyfakefs.TestCase):
+    @patch.object(ConfigurationLoader, "load", return_value={SONAR_TOKEN: "myToken", SONAR_PROJECT_KEY: "myProjectKey"})
+    @patch.object(ScannerEngine, "run", return_value=0)
+    def test_minimal_success_run(self, load_mock, run_mock):
         exitcode = scan()
         self.assertEqual(exitcode, 0)
-
-    @patch("pysonar_scanner.scannerengine.Popen")
-    @patch.object(ScannerEngine, "_ScannerEngine__resolve_jre")
-    @patch.object(ScannerEngine, "_ScannerEngine__fetch_scanner_engine")
-    @patch(
-        "sys.argv",
-        ["pysonar-scanner", "-t", "myToken", "--sonar-project-key", "myProjectKey"],
-    )
-    def test_minimal_run_failure(self, mock_fetch_scanner_engine, mock_resolve_jre, mock_popen):
-        mock_resolve_jre.return_value = JREResolvedPath("")
-        process_mock = mock.Mock()
-        mock_fetch_scanner_engine.return_value = None
-        attrs = {"communicate.return_value": ("output", "error"), "wait.return_value": 2}
-        process_mock.configure_mock(**attrs)
-        mock_popen.return_value = process_mock
-        with self.assertRaises(RuntimeError) as error:
-            scan()
-        print(str(error))
-        self.assertIn("Scan failed with exit code 2", str(error.exception))