diff --git a/build.mill b/build.mill index 23a0fbea24..988186d7a4 100644 --- a/build.mill +++ b/build.mill @@ -587,6 +587,8 @@ trait Core extends ScalaCliCrossSbtModule | def mavenAppVersion = "${Deps.Versions.mavenAppVersion}" | | def scalafixVersion = "${Deps.Versions.scalafix}" + | + | def slothVersion = "${Deps.Versions.sloth}" | | def alpineVersion = "$alpineVersion" |} @@ -744,6 +746,7 @@ trait Build extends ScalaCliCrossSbtModule Deps.nativeTestRunner, Deps.osLib, Deps.pprint, + Deps.slothCore, Deps.scalaJsEnvNodeJs, Deps.scalaJsTestAdapter, Deps.swoval, diff --git a/modules/build/src/main/scala/scala/build/postprocessing/SlothPatcher.scala b/modules/build/src/main/scala/scala/build/postprocessing/SlothPatcher.scala new file mode 100644 index 0000000000..e79a01cf08 --- /dev/null +++ b/modules/build/src/main/scala/scala/build/postprocessing/SlothPatcher.scala @@ -0,0 +1,87 @@ +package scala.build.postprocessing + +import os.Path +import sloth.jar.JarProcessor + +import java.io.{ByteArrayOutputStream, PrintStream} +import java.math.BigInteger +import java.security.MessageDigest + +import scala.build.errors.BuildException +import scala.build.internal.Constants +import scala.build.options.BuildOptions +import scala.build.{Directories, Logger} +import scala.util.control.NonFatal + +object SlothPatcher { + + private val cacheDir = Directories.directories.cacheDir / "sloth" + + def transformClassPath( + classPath: Seq[Path], + options: BuildOptions, + logger: Logger + ): Either[BuildException, Seq[Path]] = + if options.notForBloopOptions.sloth then + Right { + // sloth logs to stdout/stderr via its own bundled logging; capture it so it + // doesn't pollute the user program's output, routing anything it prints to debug. + captureStdio(logger) { + classPath.map { entry => + if entry.ext == "jar" then patchJar(entry, logger) + else entry + } + } + } + else Right(classPath) + + private def captureStdio[T](logger: Logger)(f: => T): T = { + val outBuffer = new ByteArrayOutputStream + val errBuffer = new ByteArrayOutputStream + val originalOut = System.out + val originalErr = System.err + System.setOut(new PrintStream(outBuffer, true)) + System.setErr(new PrintStream(errBuffer, true)) + try f + finally { + System.setOut(originalOut) + System.setErr(originalErr) + val captured = (outBuffer.toString ++ errBuffer.toString).trim + if captured.nonEmpty then logger.debug(captured) + } + } + + private def patchJar(jar: Path, logger: Logger): Path = + val cachedDir = cacheDir / Constants.slothVersion / sha1(jar) + val cached = cachedDir / jar.last + if os.exists(cached) then cached + else + os.makeDir.all(cachedDir) + runJarProcessor(jar, cached) match + case Right(()) => + logger.debug(s"Patched lazy vals in $jar -> $cached") + cached + case Left(message) => + os.remove.all(cachedDir) + logger.message(s"Could not patch lazy vals in $jar, using original: $message") + jar + + private def runJarProcessor( + input: Path, + output: Path + ): Either[String, Unit] = + try + val result = JarProcessor.process(input.toNIO, output.toNIO) + if result.failedClasses > 0 then + val details = if result.errors.nonEmpty then s": ${result.errors.mkString("; ")}" else "" + Left(s"Failed to patch lazy vals in $input ($result.failedClasses failed classes)$details") + else Right(()) + catch + case NonFatal(e) => + Left(s"Failed to patch lazy vals in $input: ${e.getMessage}") + + private def sha1(path: Path): String = + val md = MessageDigest.getInstance("SHA-1") + md.update(os.read.bytes(path)) + String.format("%040x", new BigInteger(1, md.digest())) +} diff --git a/modules/build/src/main/scala/scala/build/preprocessing/directives/DirectivesPreprocessingUtils.scala b/modules/build/src/main/scala/scala/build/preprocessing/directives/DirectivesPreprocessingUtils.scala index dfacd593fa..6eb1efaf7a 100644 --- a/modules/build/src/main/scala/scala/build/preprocessing/directives/DirectivesPreprocessingUtils.scala +++ b/modules/build/src/main/scala/scala/build/preprocessing/directives/DirectivesPreprocessingUtils.scala @@ -31,6 +31,7 @@ object DirectivesPreprocessingUtils { directives.ScalaNative.handler, directives.ScalaVersion.handler, directives.Sources.handler, + directives.Sloth.handler, directives.Watching.handler, directives.Tests.handler, directives.Wasm.handler diff --git a/modules/build/src/test/scala/scala/build/tests/DirectiveTests.scala b/modules/build/src/test/scala/scala/build/tests/DirectiveTests.scala index c9d233f834..e3b50efe68 100644 --- a/modules/build/src/test/scala/scala/build/tests/DirectiveTests.scala +++ b/modules/build/src/test/scala/scala/build/tests/DirectiveTests.scala @@ -518,4 +518,16 @@ class DirectiveTests extends TestUtil.ScalaCliBuildSuite { (_, _, maybeBuild) => expect(maybeBuild.exists(_.success)) } } + + test("sloth directive") { + val testInputs = TestInputs( + os.rel / "simple.sc" -> + """//> using sloth + |""".stripMargin + ) + testInputs.withBuild(baseOptions, buildThreads, bloopConfigOpt) { (_, _, maybeBuild) => + val build = maybeBuild.orThrow + expect(build.options.notForBloopOptions.slothOpt.contains(true)) + } + } } diff --git a/modules/cli/src/main/scala/scala/cli/commands/repl/Repl.scala b/modules/cli/src/main/scala/scala/cli/commands/repl/Repl.scala index 99041df486..cd734ef752 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/repl/Repl.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/repl/Repl.scala @@ -13,6 +13,7 @@ import scala.build.input.Inputs import scala.build.internal.{Constants, Runner} import scala.build.options.ScalacOpt.{filterScalacOptionKeys, noDashPrefixes} import scala.build.options.{BuildOptions, JavaOpt, Scope} +import scala.build.postprocessing.SlothPatcher import scala.cli.CurrentParams import scala.cli.commands.run.Run.{createPythonInstance, orPythonDetectionError, pythonPathEnv} import scala.cli.commands.run.RunMode @@ -346,6 +347,11 @@ object Repl extends ScalaCommand[ReplOptions] with BuildCommandHelpers { val additionalArgs = pythonReplArgs ++ options.scalaOptions.scalacOptions.toSeq.map(_.value.value) + val javaCommand = options.javaHome().value.javaCommand + + def patchClassPath(classPath: Seq[os.Path]): Seq[os.Path] = + value(SlothPatcher.transformClassPath(classPath, options, logger)) + def maybeRunRepl( replArtifacts: ReplArtifacts, replArgs: Seq[String], @@ -354,22 +360,23 @@ object Repl extends ScalaCommand[ReplOptions] with BuildCommandHelpers { ): Unit = { if dryRun then logger.message("Dry run, not running REPL.") else { + val depClassPath = patchClassPath(mainJarsOrClassDirs ++ replArtifacts.depsClassPath) + val replClassPath = patchClassPath(replArtifacts.replClassPath) val depClassPathArgs: Seq[String] = - if mainJarsOrClassDirs.nonEmpty || replArtifacts.depsClassPath.nonEmpty + if depClassPath.nonEmpty then Seq( "-classpath", - (mainJarsOrClassDirs ++ replArtifacts.depsClassPath) - .map(_.toString).mkString(File.pathSeparator) + depClassPath.map(_.toString).mkString(File.pathSeparator) ) else Nil val retCode = Runner.runJvm( - javaCommand = options.javaHome().value.javaCommand, + javaCommand = javaCommand, javaArgs = scalapyJavaOpts ++ replArtifacts.replJavaOpts ++ options.javaOptions.javaOpts.toSeq.map(_.value.value) ++ extraProps.toVector.sorted.map { case (k, v) => s"-D$k=$v" }, - classPath = replArtifacts.replClassPath, + classPath = replClassPath, mainClass = replArtifacts.replMainClass, args = maybeAdaptForWindows(depClassPathArgs ++ replArgs), logger = logger, @@ -398,12 +405,14 @@ object Repl extends ScalaCommand[ReplOptions] with BuildCommandHelpers { } } if shouldUseJshell then - val javaHomeInfo = options.javaHome().value + val javaHomeInfo = options.javaHome().value + val jshellClassPath = + patchClassPath(mainJarsOrClassDirs ++ allArtifacts.flatMap(_.classPath).distinct) val jshellCommand0 = value( JShellRunner.commandFor( javaHomeInfo = javaHomeInfo, javaOpts = scalapyJavaOpts ++ options.javaOptions.javaOpts.toSeq.map(_.value.value), - classPath = mainJarsOrClassDirs ++ allArtifacts.flatMap(_.classPath).distinct, + classPath = jshellClassPath, programArgs = programArgs, initScriptOpt = initScriptOpt, quitAfterInit = quitAfterInit, diff --git a/modules/cli/src/main/scala/scala/cli/commands/run/Run.scala b/modules/cli/src/main/scala/scala/cli/commands/run/Run.scala index d63ce9f89e..83c2034143 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/run/Run.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/run/Run.scala @@ -19,6 +19,7 @@ import scala.build.internals.ConsoleUtils.ScalaCliConsole import scala.build.internals.ConsoleUtils.ScalaCliConsole.warnPrefix import scala.build.internals.EnvVar import scala.build.options.{BuildOptions, JSRuntime, JavaOpt, PackageType, Platform, Scope} +import scala.build.postprocessing.SlothPatcher import scala.cli.CurrentParams import scala.cli.commands.package0.Package import scala.cli.commands.setupide.SetupIde @@ -689,6 +690,14 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers { ++ Seq(s"-Dscala.sources=$sources", s"-Dscala.source.names=$sourceNames") val setupPython = build.options.notForBloopOptions.doSetupPython.getOrElse(false) + val classPath0 = builds.flatMap(_.fullClassPathMaybeAsJar(asJar)).distinct + val classPath = value( + SlothPatcher.transformClassPath( + classPath0, + build.options, + logger + ) + ) val (pythonJavaProps, pythonExtraEnv) = if setupPython then { val scalapyProps = value { @@ -713,7 +722,7 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers { Runner.jvmCommand( build.options.javaHome().value.javaCommand, allJavaOpts, - builds.flatMap(_.fullClassPathMaybeAsJar(asJar)).distinct, + classPath, mainClass, args, extraEnv = pythonExtraEnv, @@ -725,7 +734,7 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers { val proc = Runner.runJvm( javaCommand = build.options.javaHome().value.javaCommand, javaArgs = allJavaOpts, - classPath = builds.flatMap(_.fullClassPathMaybeAsJar(asJar)).distinct, + classPath = classPath, mainClass = mainClass, args = args, logger = logger, diff --git a/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala index 2db2f5e40f..4d5ae0c8af 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala @@ -210,6 +210,11 @@ final case class SharedOptions( @HelpMessage("Force object wrapper for scripts") @Tag(tags.experimental) objectWrapper: Option[Boolean] = None, + @HelpMessage("Patch Scala 3.0-3.7.x lazy val bytecode on the classpath for JDK 26+ compatibility") + @Name("lazyvalgrade") + @Name("patch-lazy-vals") + @Tag(tags.experimental) + sloth: Option[Boolean] = None, @Group(HelpGroup.Scala.toString) @HelpMessage( "Automatically generate BSP configuration in `.bsp/` when running build commands. Enabled by default." @@ -490,7 +495,8 @@ final case class SharedOptions( addRunnerDependencyOpt = runner, python = sharedPython.python, pythonSetup = sharedPython.pythonSetup, - scalaPyVersion = sharedPython.scalaPyVersion + scalaPyVersion = sharedPython.scalaPyVersion, + slothOpt = sloth ), useBuildServer = compilationServer.server ).orElse(watchOptions.buildOptions()) diff --git a/modules/cli/src/test/scala/cli/commands/tests/RunOptionsTests.scala b/modules/cli/src/test/scala/cli/commands/tests/RunOptionsTests.scala index 6aaa38a03b..6a0d9b6f44 100644 --- a/modules/cli/src/test/scala/cli/commands/tests/RunOptionsTests.scala +++ b/modules/cli/src/test/scala/cli/commands/tests/RunOptionsTests.scala @@ -50,4 +50,14 @@ class RunOptionsTests extends munit.FunSuite { expect(toolkitDep.name == "toolkit") expect(toolkitDep.version == "latest.release") } + + test("sloth option") { + val runOptions = RunOptions( + shared = SharedOptions( + sloth = Some(true) + ) + ) + val buildOptions = Run.buildOptions(runOptions).value + expect(buildOptions.notForBloopOptions.slothOpt.contains(true)) + } } diff --git a/modules/directives/src/main/scala/scala/build/preprocessing/directives/Sloth.scala b/modules/directives/src/main/scala/scala/build/preprocessing/directives/Sloth.scala new file mode 100644 index 0000000000..456252bae0 --- /dev/null +++ b/modules/directives/src/main/scala/scala/build/preprocessing/directives/Sloth.scala @@ -0,0 +1,27 @@ +package scala.build.preprocessing.directives + +import scala.build.directives.* +import scala.build.errors.BuildException +import scala.build.options.{BuildOptions, PostBuildOptions} +import scala.cli.commands.SpecificationLevel + +@DirectiveExamples("//> using sloth") +@DirectiveUsage("//> using sloth", "`//> using sloth`") +@DirectiveDescription( + "Patch Scala 3.0-3.7.x lazy val bytecode on the classpath for JDK 26+ compatibility" +) +@DirectiveLevel(SpecificationLevel.EXPERIMENTAL) +final case class Sloth( + @DirectiveName("lazyvalgrade") + @DirectiveName("lazyValPatching") + sloth: Boolean = false +) extends HasBuildOptions { + def buildOptions: Either[BuildException, BuildOptions] = + Right(BuildOptions( + notForBloopOptions = PostBuildOptions(slothOpt = Some(true)) + )) +} + +object Sloth { + val handler: DirectiveHandler[Sloth] = DirectiveHandler.derive +} diff --git a/modules/integration/src/test/scala/scala/cli/integration/LazyValTests.scala b/modules/integration/src/test/scala/scala/cli/integration/LazyValTests.scala new file mode 100644 index 0000000000..b423a88586 --- /dev/null +++ b/modules/integration/src/test/scala/scala/cli/integration/LazyValTests.scala @@ -0,0 +1,43 @@ +package scala.cli.integration + +trait LazyValTests { + private val lazyValsLibOrg = "test.lazyvals" + private val lazyValsLibName = "lazyvals-lib" + private val lazyValsLibVersion = "0.1.0" + private val lazyValsLibDep = s"$lazyValsLibOrg::$lazyValsLibName:$lazyValsLibVersion" + + protected def publishLazyValsLib( + scalaVersion: String, + workspace: os.Path + ): (String, os.Path) = { + val libDir = workspace / "lazyvals-lib" + val repoDir = workspace / "test-repo" + os.write( + libDir / "LazyValsLib.scala", + """package lazyvalslib + |object LazyValsLib { + | lazy val greeting: String = "Hello" + |} + |""".stripMargin, + createFolders = true + ) + os.proc( + TestUtil.cli, + "--power", + "publish", + libDir, + "--organization", + lazyValsLibOrg, + "--name", + lazyValsLibName, + "--project-version", + lazyValsLibVersion, + "--scala", + scalaVersion, + "--publish-repo", + repoDir.toNIO.toUri.toASCIIString + ).call(cwd = workspace, stdin = os.Inherit, stdout = os.Inherit) + os.remove.all(libDir) + (lazyValsLibDep, repoDir) + } +} diff --git a/modules/integration/src/test/scala/scala/cli/integration/ReplTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/ReplTestDefinitions.scala index 735e09c038..bb9cf9683c 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/ReplTestDefinitions.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/ReplTestDefinitions.scala @@ -5,7 +5,8 @@ import com.eed3si9n.expecty.Expecty.expect import scala.cli.integration.TestUtil.removeAnsiColors import scala.util.Properties -abstract class ReplTestDefinitions extends ScalaCliSuite with TestScalaVersionArgs { +abstract class ReplTestDefinitions extends ScalaCliSuite with TestScalaVersionArgs + with LazyValTests { this: TestScalaVersion => protected lazy val extraOptions: Seq[String] = scalaVersionArgs ++ TestUtil.extraOptions @@ -334,6 +335,59 @@ abstract class ReplTestDefinitions extends ScalaCliSuite with TestScalaVersionAr ) } + if isScala38OrNewer then { + val latestJava = Constants.allJavaVersions.max.toString + test( + s"$runInReplPrefix dont warn about sun.misc.Unsafe on JDK $latestJava (no dependency)" + ) { + val expectedMessage = "Hello" + val code = s"""println("$expectedMessage")""" + TestInputs.empty.fromRoot { root => + val res = os.proc( + TestUtil.cli, + "repl", + "--repl-quit-after-init", + "--repl-init-script", + code, + "--jvm", + latestJava, + "--power", + "--sloth", + extraOptions + ).call(cwd = root, stderr = os.Pipe) + expect(res.out.trim().contains(expectedMessage)) + expect(!res.err.trim().contains("sun.misc.Unsafe")) + } + } + + test(s"$runInReplPrefix 3.3 lazy vals dont warn about sun.misc.Unsafe on JDK $latestJava") { + val expectedMessage = "Hello" + val code = "println(lazyvalslib.LazyValsLib.greeting)" + TestInputs.empty.fromRoot { root => + val (dep, repoDir) = publishLazyValsLib(Constants.scala3Lts, root) + val res = os.proc( + TestUtil.cli, + "repl", + ".", + "--repl-quit-after-init", + "--repl-init-script", + code, + extraOptions, + "--power", + "--sloth", + "--dep", + dep, + "--repository", + repoDir.toNIO.toUri.toASCIIString, + "--jvm", + latestJava + ).call(cwd = root, stderr = os.Pipe) + expect(res.out.trim().contains(expectedMessage)) + expect(!res.err.trim().contains("sun.misc.Unsafe")) + } + } + } + if !isScala38OrNewer then // TODO rewrite this test to work with Scala 3.8+ once 3.8.0 stable is out test(s"$runInReplPrefix as jar") { diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala index 295bdd5c87..b7265018b9 100755 --- a/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala @@ -23,7 +23,8 @@ abstract class RunTestDefinitions with RunScalaPyTestDefinitions with RunZipTestDefinitions with RunJdkTestDefinitions - with CoursierScalaInstallationTestHelper { this: TestScalaVersion => + with CoursierScalaInstallationTestHelper + with LazyValTests { this: TestScalaVersion => protected lazy val extraOptions: Seq[String] = scalaVersionArgs ++ TestUtil.extraOptions protected val emptyInputs: TestInputs = TestInputs(os.rel / ".placeholder" -> "") @@ -2568,4 +2569,41 @@ abstract class RunTestDefinitions expect(output == message) } } + + if isScala38OrNewer then { + val latestJava = Constants.allJavaVersions.max + + def lazyValsUnsafeTest(libScalaVersion: String): Unit = + test(s"$libScalaVersion lazy vals dont warn about sun.misc.Unsafe on JDK $latestJava") { + val expectedMessage = "Hello" + TestInputs.empty.fromRoot { root => + val (dep, repoDir) = publishLazyValsLib(libScalaVersion, root) + os.write( + root / "script.sc", + s"""//> using dep $dep + |println(lazyvalslib.LazyValsLib.greeting) + |""".stripMargin + ) + val r = os.proc( + TestUtil.cli, + extraOptions, + "--power", + "--sloth", + "script.sc", + "--repository", + repoDir.toNIO.toUri.toASCIIString, + "--jvm", + latestJava + ).call(cwd = root, stderr = os.Pipe) + expect(r.out.trim() == expectedMessage) + expect(!r.err.trim().contains("sun.misc.Unsafe")) + } + } + + val highest30 = Constants.legacyScala3Versions + .filter(_.startsWith("3.0.")) + .maxBy(_.coursierVersion) + lazyValsUnsafeTest(highest30) + lazyValsUnsafeTest(Constants.scala3Lts) + } } diff --git a/modules/options/src/main/scala/scala/build/options/PostBuildOptions.scala b/modules/options/src/main/scala/scala/build/options/PostBuildOptions.scala index f3bca301ed..181598e7df 100644 --- a/modules/options/src/main/scala/scala/build/options/PostBuildOptions.scala +++ b/modules/options/src/main/scala/scala/build/options/PostBuildOptions.scala @@ -11,11 +11,15 @@ final case class PostBuildOptions( pythonSetup: Option[Boolean] = None, python: Option[Boolean] = None, scalaPyVersion: Option[String] = None, - addRunnerDependencyOpt: Option[Boolean] = None + addRunnerDependencyOpt: Option[Boolean] = None, + slothOpt: Option[Boolean] = None ) { def doSetupPython: Option[Boolean] = pythonSetup.orElse(python) + + def sloth: Boolean = + slothOpt.getOrElse(false) } object PostBuildOptions { diff --git a/project/deps/package.mill b/project/deps/package.mill index 17b633e042..2b0aae1479 100644 --- a/project/deps/package.mill +++ b/project/deps/package.mill @@ -151,8 +151,13 @@ object Deps { def mavenAppGroupId = "com.example" def mavenAppVersion = "0.1-SNAPSHOT" def scalafix = "0.14.7" + def sloth = "0.1.0-M1" } + def slothOrganization = "org.virtuslab" + def slothModuleName = "sloth-core" + def slothCore = mvn"org.virtuslab::sloth-core:${Versions.sloth}" + def argonautShapeless = mvn"com.github.alexarchambault:argonaut-shapeless_6.3_2.13:${Versions.argonautShapeless}" def asm = mvn"org.ow2.asm:asm:9.10.1" diff --git a/website/docs/reference/cli-options.md b/website/docs/reference/cli-options.md index da37782a80..f25f6b84b5 100644 --- a/website/docs/reference/cli-options.md +++ b/website/docs/reference/cli-options.md @@ -1708,6 +1708,12 @@ Exclude sources Force object wrapper for scripts +### `--sloth` + +Aliases: `--lazyvalgrade`, `--patch-lazy-vals` + +Patch Scala 3.0-3.7.x lazy val bytecode on the classpath for JDK 26+ compatibility + ### `--auto-setup-ide` Aliases: `--auto-setup-bsp` diff --git a/website/docs/reference/directives.md b/website/docs/reference/directives.md index 761efc429b..cd9ec43afb 100644 --- a/website/docs/reference/directives.md +++ b/website/docs/reference/directives.md @@ -680,6 +680,15 @@ Add Scala.js options `//> using jsEmitWasm` +### Sloth + +Patch Scala 3.0-3.7.x lazy val bytecode on the classpath for JDK 26+ compatibility + +`//> using sloth` + +#### Examples +`//> using sloth` + ### Test framework Set the test framework