SCANPY-139 Parse well-known project properties form pyproject.toml (#160)

Author: guillaume-dequenne

src/pysonar_scanner/configuration/cli.py

@@ -213,4 +213,25 @@ def __parse_cli_args(cls) -> argparse.Namespace:
             help="Path to the pyproject.toml file. If not provided, it will look in the SONAR_PROJECT_BASE_DIR",
         )
 
+        parser.add_argument(
+            "--sonar-project-version",
+            "-Dsonar.projectVersion",
+            type=str,
+            help="Version of the project",
+        )
+
+        parser.add_argument(
+            "--sonar-project-description",
+            "-Dsonar.projectDescription",
+            type=str,
+            help="Description of the project",
+        )
+
+        parser.add_argument(
+            "--sonar-python-version",
+            "-Dsonar.python.version",
+            type=str,
+            help="Python version used for the project",
+        )
+
         return parser.parse_args()

src/pysonar_scanner/configuration/configuration_loader.py

@@ -20,9 +20,10 @@
 from pathlib import Path
 
 from pysonar_scanner.configuration.cli import CliConfigurationLoader
+from pysonar_scanner.configuration.pyproject_toml import TomlConfigurationLoader
 from pysonar_scanner.configuration.properties import SONAR_TOKEN, SONAR_PROJECT_BASE_DIR, Key
 from pysonar_scanner.configuration.properties import PROPERTIES
-from pysonar_scanner.configuration import sonar_project_properties, pyproject_toml
+from pysonar_scanner.configuration import sonar_project_properties
 
 from pysonar_scanner.exceptions import MissingKeyException
 
@@ -37,17 +38,19 @@ def load() -> dict[Key, any]:
         # each property loader is required to return NO default values.
         # E.g. if no property has been set, an empty dict must be returned.
         # Default values should be set through the get_static_default_properties() method
-        resolved_properties = get_static_default_properties()
         cli_properties = CliConfigurationLoader.load()
         # CLI properties have a higher priority than properties file,
         # but we need to resolve them first to load the properties file
         base_dir = Path(cli_properties.get(SONAR_PROJECT_BASE_DIR, "."))
-        resolved_properties.update(sonar_project_properties.load(base_dir))
 
         toml_path_property = cli_properties.get("toml-path", ".")
         toml_dir = Path(toml_path_property) if "toml-path" in cli_properties else base_dir
-        resolved_properties.update(pyproject_toml.load(toml_dir))
+        toml_properties = TomlConfigurationLoader.load(toml_dir)
 
+        resolved_properties = get_static_default_properties()
+        resolved_properties.update(toml_properties.project_properties)
+        resolved_properties.update(sonar_project_properties.load(base_dir))
+        resolved_properties.update(toml_properties.sonar_properties)
         resolved_properties.update(cli_properties)
         return resolved_properties
 

src/pysonar_scanner/configuration/properties.py

@@ -60,6 +60,9 @@
 SONAR_SOURCES: Key = "sonar.sources"
 SONAR_EXCLUSIONS: Key = "sonar.exclusions"
 SONAR_TESTS: Key = "sonar.tests"
+SONAR_PROJECT_VERSION: Key = "sonar.projectVersion"
+SONAR_PROJECT_DESCRIPTION: Key = "sonar.projectDescription"
+SONAR_PYTHON_VERSION: Key = "sonar.python.version"
 
 # pysonar scanner specific properties
 TOML_PATH: Key = "toml-path"
@@ -258,5 +261,20 @@ def python_name(self) -> str:
             default_value=None,
             cli_getter=lambda args: args.toml_path
     ),
+    Property(
+                name=SONAR_PROJECT_VERSION,
+                default_value=None,
+                cli_getter=lambda args: args.sonar_project_version
+    ),
+    Property(
+                name=SONAR_PROJECT_DESCRIPTION,
+                default_value=None,
+                cli_getter=lambda args: args.sonar_project_description
+    ),
+    Property(
+                name=SONAR_PYTHON_VERSION,
+                default_value=None,
+                cli_getter=lambda args: args.sonar_python_version
+    ),
 ]
 # fmt: on

src/pysonar_scanner/configuration/pyproject_toml.py

@@ -25,35 +25,79 @@
 from pysonar_scanner.configuration import properties
 
 
-def flatten_config_dict(config: dict[str, any], prefix: str) -> dict[str, any]:
-    """Flatten nested dictionaries into dot notation keys"""
-    result = {}
-    for key, value in config.items():
-        if isinstance(value, dict):
-            result.update(flatten_config_dict(value, f"{prefix}{key}."))
-        else:
-            property_name = f"{prefix}{key}"
-            result[property_name] = value
-    return result
+class TomlProperties:
+    sonar_properties: Dict[str, str]
+    project_properties: Dict[str, str]
 
+    def __init__(self, sonar_properties: Dict[str, str], project_properties: Dict[str, str]):
+        self.sonar_properties = sonar_properties
+        self.project_properties = project_properties
 
-def load(base_dir: Path) -> Dict[str, str]:
-    filepath = base_dir / "pyproject.toml"
-    if not os.path.isfile(filepath):
-        return {}
 
-    try:
-        with open(filepath, "rb") as f:
-            toml_dict = tomli.load(f)
+class TomlConfigurationLoader:
+    @staticmethod
+    def load(base_dir: Path) -> TomlProperties:
+        filepath = base_dir / "pyproject.toml"
+        if not os.path.isfile(filepath):
+            return TomlProperties({}, {})
+
+        try:
+            with open(filepath, "rb") as f:
+                toml_dict = tomli.load(f)
+            # Look for configuration in the tool.sonar section
+            sonar_properties = TomlConfigurationLoader.__read_sonar_properties(toml_dict)
+            # Look for general project configuration
+            project_properties = TomlConfigurationLoader.__read_project_properties(toml_dict)
+            return TomlProperties(sonar_properties, project_properties)
+        except Exception:
+            # If there's any error parsing the TOML file, return empty TomlProperties
+            # SCANPY-135: We should log the pyproject.toml parsing error
+            return TomlProperties({}, {})
 
-        # Look for configuration in the tool.sonar section
+    @staticmethod
+    def __read_sonar_properties(toml_dict) -> Dict[str, str]:
         if "tool" in toml_dict and "sonar" in toml_dict["tool"]:
             sonar_config = toml_dict["tool"]["sonar"]
             python_to_java_names = {prop.python_name(): prop.name for prop in properties.PROPERTIES}
-            flattened_sonar_config = flatten_config_dict(sonar_config, prefix="sonar.")
+            flattened_sonar_config = TomlConfigurationLoader.__flatten_config_dict(sonar_config, prefix="sonar.")
             return {python_to_java_names.get(key, key): value for key, value in flattened_sonar_config.items()}
         return {}
-    except Exception:
-        # If there's any error parsing the TOML file, return empty dict
-        # SCANPY-135: We should log the pyproject.toml parsing error
-        return {}
+
+    @staticmethod
+    def __read_project_properties(toml_dict) -> Dict[str, str]:
+        # Extract project metadata
+        project_properties = {}
+        if "project" in toml_dict:
+            project_data = toml_dict["project"]
+            # Known pyproject.toml project keys and associated Sonar property names
+            property_mapping = {
+                "name": properties.SONAR_PROJECT_NAME,
+                "description": properties.SONAR_PROJECT_DESCRIPTION,
+                "version": properties.SONAR_PROJECT_VERSION,
+                "requires-python": properties.SONAR_PYTHON_VERSION,
+            }
+
+            for toml_key, sonar_property in property_mapping.items():
+                if toml_key in project_data:
+                    string_property = TomlConfigurationLoader.__convert_arrays_to_string(project_data[toml_key])
+                    project_properties[sonar_property] = string_property
+        return project_properties
+
+    @staticmethod
+    def __flatten_config_dict(config: dict[str, any], prefix: str) -> dict[str, any]:
+        """Flatten nested dictionaries into dot notation keys"""
+        result = {}
+        for key, value in config.items():
+            if isinstance(value, dict):
+                result.update(TomlConfigurationLoader.__flatten_config_dict(value, f"{prefix}{key}."))
+            else:
+                property_name = f"{prefix}{key}"
+                result[property_name] = value
+        return result
+
+    @staticmethod
+    def __convert_arrays_to_string(property) -> str:
+        if isinstance(property, list):
+            return ",".join(str(item) for item in property)
+        else:
+            return property

tests/test_configuration_loader.py

@@ -42,6 +42,8 @@
     SONAR_USER_HOME,
     SONAR_VERBOSE,
     TOML_PATH,
+    SONAR_PROJECT_DESCRIPTION,
+    SONAR_PYTHON_VERSION,
 )
 from pysonar_scanner.configuration.configuration_loader import ConfigurationLoader, SONAR_PROJECT_BASE_DIR
 from pysonar_scanner.exceptions import MissingKeyException
@@ -292,6 +294,10 @@ def test_properties_and_toml_priority(self):
             "pyproject.toml",
             contents=(
                 """
+                [project]
+                name = "My Overridden Project Name"
+                description = "My Project Description"
+                requires-python = ["3.6", "3.7", "3.8"]
                 [tool.sonar]
                 projectKey = "toml-project-key"
                 project-name = "TOML Project"
@@ -303,11 +309,17 @@ def test_properties_and_toml_priority(self):
 
         configuration = ConfigurationLoader.load()
 
-        # TOML values have priority over sonar-project.properties
+        # Generic pyproject.toml properties are retrieved when no other source is available
+        self.assertEqual(configuration[SONAR_PROJECT_DESCRIPTION], "My Project Description")
+        self.assertEqual(configuration[SONAR_PYTHON_VERSION], "3.6,3.7,3.8")
+
+        # sonar-project.properties values have priority over generic properties from pyproject.toml
         self.assertEqual(configuration[SONAR_PROJECT_NAME], "TOML Project")
+
+        # Sonar pyproject.toml values have priority over sonar-project.properties
         self.assertEqual(configuration[SONAR_SOURCES], "src/toml")
-        self.assertEqual(configuration[SONAR_TESTS], "test/properties")
         self.assertEqual(configuration[SONAR_EXCLUSIONS], "toml-exclusions/**/*")
 
         # CLI args still have highest priority
         self.assertEqual(configuration[SONAR_PROJECT_KEY], "ProjectKeyFromCLI")
+        self.assertEqual(configuration[SONAR_TESTS], "test/properties")

tests/test_pyproject_toml.py

@@ -20,7 +20,7 @@
 from pathlib import Path
 
 from pyfakefs.fake_filesystem_unittest import TestCase
-from pysonar_scanner.configuration import pyproject_toml
+from pysonar_scanner.configuration.pyproject_toml import TomlConfigurationLoader
 
 
 class TestTomlFile(TestCase):
@@ -38,12 +38,14 @@ def test_load_toml_file_with_sonarqube_config(self):
             exclusions = "**/generated/**/*,**/deprecated/**/*,**/testdata/**/*"
             """,
         )
-        properties = pyproject_toml.load(Path("."))
+        properties = TomlConfigurationLoader.load(Path("."))
 
-        self.assertEqual(properties.get("sonar.projectKey"), "my-project")
-        self.assertEqual(properties.get("sonar.projectName"), "My Project")
-        self.assertEqual(properties.get("sonar.sources"), "src")
-        self.assertEqual(properties.get("sonar.exclusions"), "**/generated/**/*,**/deprecated/**/*,**/testdata/**/*")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectKey"), "my-project")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectName"), "My Project")
+        self.assertEqual(properties.sonar_properties.get("sonar.sources"), "src")
+        self.assertEqual(
+            properties.sonar_properties.get("sonar.exclusions"), "**/generated/**/*,**/deprecated/**/*,**/testdata/**/*"
+        )
 
     def test_load_toml_file_kebab_case(self):
         self.fs.create_file(
@@ -54,10 +56,10 @@ def test_load_toml_file_kebab_case(self):
             project-name = "My Project"
             """,
         )
-        properties = pyproject_toml.load(Path("."))
+        properties = TomlConfigurationLoader.load(Path("."))
 
-        self.assertEqual(properties.get("sonar.projectKey"), "my-project")
-        self.assertEqual(properties.get("sonar.projectName"), "My Project")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectKey"), "my-project")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectName"), "My Project")
 
     def test_load_toml_file_without_sonar_section(self):
         self.fs.create_file(
@@ -71,19 +73,19 @@ def test_load_toml_file_without_sonar_section(self):
             profile = "black"
             """,
         )
-        properties = pyproject_toml.load(Path("."))
+        properties = TomlConfigurationLoader.load(Path("."))
 
-        self.assertEqual(len(properties), 0)
+        self.assertEqual(len(properties.sonar_properties), 0)
 
     def test_load_missing_file(self):
-        properties = pyproject_toml.load(Path("."))
-        self.assertEqual(len(properties), 0)
+        properties = TomlConfigurationLoader.load(Path("."))
+        self.assertEqual(len(properties.sonar_properties), 0)
 
     def test_load_empty_file(self):
         self.fs.create_file("pyproject.toml", contents="")
-        properties = pyproject_toml.load(Path("."))
+        properties = TomlConfigurationLoader.load(Path("."))
 
-        self.assertEqual(len(properties), 0)
+        self.assertEqual(len(properties.sonar_properties), 0)
 
     def test_load_malformed_toml_file(self):
         self.fs.create_file(
@@ -93,9 +95,9 @@ def test_load_malformed_toml_file(self):
             sonar.projectKey = "my-project"
             """,
         )
-        properties = pyproject_toml.load(Path("."))
+        properties = TomlConfigurationLoader.load(Path("."))
 
-        self.assertEqual(len(properties), 0)
+        self.assertEqual(len(properties.sonar_properties), 0)
 
     def test_load_toml_with_nested_values(self):
         self.fs.create_file(
@@ -109,11 +111,11 @@ def test_load_toml_with_nested_values(self):
             coverage.reportPaths = "coverage.xml"
             """,
         )
-        properties = pyproject_toml.load(Path("."))
+        properties = TomlConfigurationLoader.load(Path("."))
 
-        self.assertEqual(properties.get("sonar.projectKey"), "my-project")
-        self.assertEqual(properties.get("sonar.python.version"), "3.9,3.10,3.11,3.12,3.13")
-        self.assertEqual(properties.get("sonar.python.coverage.reportPaths"), "coverage.xml")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectKey"), "my-project")
+        self.assertEqual(properties.sonar_properties.get("sonar.python.version"), "3.9,3.10,3.11,3.12,3.13")
+        self.assertEqual(properties.sonar_properties.get("sonar.python.coverage.reportPaths"), "coverage.xml")
 
     def test_load_toml_file_from_custom_dir(self):
         self.fs.create_dir("custom/path")
@@ -125,7 +127,30 @@ def test_load_toml_file_from_custom_dir(self):
             projectName = "Custom Path Project"
             """,
         )
-        properties = pyproject_toml.load(Path("custom/path"))
+        properties = TomlConfigurationLoader.load(Path("custom/path"))
+
+        self.assertEqual(properties.sonar_properties.get("sonar.projectKey"), "custom-path-project")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectName"), "Custom Path Project")
+
+    def test_load_toml_file_project_content(self):
+        self.fs.create_file(
+            "pyproject.toml",
+            contents=(
+                """
+                [project]
+                name = "My Overridden Project Name"
+                description = "My Project Description"
+                requires-python = ["3.6", "3.7", "3.8"]
+                [tool.sonar]
+                project-key = "my-project"
+                project-name = "My Project"
+                """
+            ),
+        )
+        properties = TomlConfigurationLoader.load(Path("."))
 
-        self.assertEqual(properties.get("sonar.projectKey"), "custom-path-project")
-        self.assertEqual(properties.get("sonar.projectName"), "Custom Path Project")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectKey"), "my-project")
+        self.assertEqual(properties.sonar_properties.get("sonar.projectName"), "My Project")
+        self.assertEqual(properties.project_properties.get("sonar.projectName"), "My Overridden Project Name")
+        self.assertEqual(properties.project_properties.get("sonar.projectDescription"), "My Project Description")
+        self.assertEqual(properties.project_properties.get("sonar.python.version"), "3.6,3.7,3.8")