diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index f4ff68f9a..877a7a98c 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -5919,6 +5919,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "tokio-rustls", "tracing", "x509-parser 0.16.0", ] diff --git a/dstack/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs index f506482a8..1046d35d1 100644 --- a/dstack/dstack-util/src/main.rs +++ b/dstack/dstack-util/src/main.rs @@ -655,14 +655,14 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { // Step 1: Fetch the KMS root CA eprintln!("Connecting to KMS: {kms_url}"); - let tls_no_check = root_ca_pem.is_none(); - if tls_no_check { + let tls_skip_ca_verification = root_ca_pem.is_none(); + if tls_skip_ca_verification { eprintln!("Warning: no --root-ca provided, TLS certificate verification is disabled for initial connection"); } let root_ca = { let client = RaClientConfig::builder() .remote_uri(kms_url.clone()) - .tls_no_check(tls_no_check) + .tls_skip_ca_verification(tls_skip_ca_verification) .tls_built_in_root_certs(false) .maybe_tls_ca_cert(root_ca_pem.clone()) .build() @@ -682,7 +682,7 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { // Step 3: Create authenticated client and request app keys let ra_client = RaClientConfig::builder() - .tls_no_check(false) + .tls_skip_ca_verification(false) .tls_built_in_root_certs(false) .remote_uri(kms_url.clone()) .tls_client_cert(cert_pair.cert_pem) @@ -819,7 +819,7 @@ async fn cmd_encrypt(args: EncryptArgs) -> Result<()> { .transpose()?; let client = RaClientConfig::builder() .remote_uri(kms_url) - .tls_no_check(false) + .tls_skip_ca_verification(false) .tls_built_in_root_certs(root_ca_pem.is_none()) .maybe_tls_ca_cert(root_ca_pem) .build() diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 4c6399cc2..0d604076b 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -592,7 +592,7 @@ impl<'a> GatewayContext<'a> { .tls_client_key(client_key.to_string()) .tls_ca_cert(ca_cert) .tls_built_in_root_certs(false) - .tls_no_check(gateway_app_id == "any") + .tls_skip_ca_verification(gateway_app_id == "any") .verify_server_attestation(false) .cert_validator(Box::new(move |cert| cert_validator.validate(cert))) .build() @@ -2437,7 +2437,7 @@ impl<'a> Stage0<'a> { generate_self_signed_ra_cert(None).context("Failed to generate RA-TLS cert")?; let attestation_verifier = attestation_verifier(&self.shared.sys_config)?; let ra_client = RaClientConfig::builder() - .tls_no_check(false) + .tls_skip_ca_verification(false) .tls_built_in_root_certs(false) .remote_uri(kms_url.clone()) .tls_client_cert(cert_pair.cert_pem) diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index 73a91a74d..480f9b62a 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -742,7 +742,7 @@ impl Keys { }; let (ra_cert, ra_key) = gen_ra_cert(tmp_ca).await?; let client = RaClientConfig::builder() - .tls_no_check(true) + .tls_skip_ca_verification(true) .tls_built_in_root_certs(false) .remote_uri(other_kms_url.to_string()) .tls_client_cert(ra_cert) diff --git a/dstack/ra-rpc/Cargo.toml b/dstack/ra-rpc/Cargo.toml index 8da9df01f..0a4516396 100644 --- a/dstack/ra-rpc/Cargo.toml +++ b/dstack/ra-rpc/Cargo.toml @@ -28,7 +28,9 @@ prost-types = { workspace = true, optional = true } or-panic.workspace = true [dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +ra-tls = { workspace = true, features = ["test-pki"] } +tokio-rustls.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util"] } reqwest = { workspace = true, default-features = false, features = [ "rustls", "charset", @@ -37,5 +39,5 @@ reqwest = { workspace = true, default-features = false, features = [ [features] default = ["rocket", "client"] rocket = ["dep:rocket", "dep:rocket-vsock-listener", "dep:rustls"] -client = ["reqwest"] +client = ["reqwest", "dep:rustls"] openapi = ["dep:prost-types"] diff --git a/dstack/ra-rpc/src/client.rs b/dstack/ra-rpc/src/client.rs index 0405d5517..b5c133fef 100644 --- a/dstack/ra-rpc/src/client.rs +++ b/dstack/ra-rpc/src/client.rs @@ -18,6 +18,8 @@ use serde::{de::DeserializeOwned, Serialize}; use bon::Builder; +mod tls; + pub struct CertInfo { pub cert_der: Vec, pub attestation: Option, @@ -30,8 +32,10 @@ type CertValidator = Box) -> Result<()> + Send + Sync + #[derive(Builder)] pub struct RaClientConfig { remote_uri: String, + /// Skip CA-based certificate checks (trust chain, hostname, and validity). + /// TLS handshake signatures remain verified; attestation is checked separately. #[builder(default = false)] - tls_no_check: bool, + tls_skip_ca_verification: bool, #[builder(default = true)] verify_server_attestation: bool, #[builder(default = false)] @@ -50,14 +54,19 @@ impl RaClientConfig { pub fn into_client(self) -> Result { let mut builder = Client::builder() .tls_sni(true) - .danger_accept_invalid_certs(self.tls_no_check) .danger_accept_invalid_hostnames(self.tls_no_check_hostname) .connect_timeout(Duration::from_secs(5)) .timeout(Duration::from_secs(60)); if self.cert_validator.is_some() { builder = builder.tls_info(true); } - if let (Some(cert_pem), Some(key_pem)) = (self.tls_client_cert, self.tls_client_key) { + if self.tls_skip_ca_verification { + builder = builder.use_preconfigured_tls(tls::config_without_ca_verification( + self.tls_client_cert.as_deref(), + self.tls_client_key.as_deref(), + )?); + } else if let (Some(cert_pem), Some(key_pem)) = (self.tls_client_cert, self.tls_client_key) + { let identity_pem = format!("{cert_pem}\n{key_pem}"); let identity = Identity::from_pem(identity_pem.as_bytes()).context("Failed to parse identity")?; @@ -104,9 +113,9 @@ pub struct RaClient { } impl RaClient { - pub fn new(remote_uri: String, tls_no_check: bool) -> Result { + pub fn new(remote_uri: String, tls_skip_ca_verification: bool) -> Result { RaClientConfig::builder() - .tls_no_check(tls_no_check) + .tls_skip_ca_verification(tls_skip_ca_verification) .remote_uri(remote_uri) .build() .into_client() diff --git a/dstack/ra-rpc/src/client/tls.rs b/dstack/ra-rpc/src/client/tls.rs new file mode 100644 index 000000000..1bb9a2809 --- /dev/null +++ b/dstack/ra-rpc/src/client/tls.rs @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; + +use anyhow::{bail, Context, Result}; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + crypto::WebPkiSupportedAlgorithms, + pki_types::{CertificateDer, ServerName, UnixTime}, + DigitallySignedStruct, SignatureScheme, +}; + +/// Attestation is checked on the response, but TLS must still prove possession +/// of the attested certificate key. Reqwest's no-verifier also skips that proof. +#[derive(Debug)] +struct SkipCaVerification(WebPkiSupportedAlgorithms); + +impl ServerCertVerifier for SkipCaVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, signature, &self.0) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, signature, &self.0) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.supported_schemes() + } +} + +pub(super) fn config_without_ca_verification( + cert: Option<&str>, + key: Option<&str>, +) -> Result { + use rustls::pki_types::{pem::PemObject, PrivateKeyDer}; + let provider = rustls::crypto::CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(rustls::crypto::aws_lc_rs::default_provider())); + let verifier = SkipCaVerification(provider.signature_verification_algorithms); + let builder = rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions()? + .dangerous() + .with_custom_certificate_verifier(Arc::new(verifier)); + match (cert, key) { + (Some(cert), Some(key)) => builder + .with_client_auth_cert( + CertificateDer::pem_slice_iter(cert.as_bytes()).collect::, _>>()?, + PrivateKeyDer::from_pem_slice(key.as_bytes())?, + ) + .context("invalid RA-TLS client identity"), + (None, None) => Ok(builder.with_no_client_auth()), + _ => bail!("RA-TLS client identity requires both certificate and key"), + } +} diff --git a/dstack/ra-rpc/tests/server_key_possession.rs b/dstack/ra-rpc/tests/server_key_possession.rs new file mode 100644 index 000000000..eeb874af7 --- /dev/null +++ b/dstack/ra-rpc/tests/server_key_possession.rs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "client")] + +use std::sync::Arc; + +use prpc::client::RequestClient; +use ra_rpc::client::RaClientConfig; +use ra_tls::test_pki::{TestCa, TestCert}; +use rustls::{ + pki_types::PrivatePkcs8KeyDer, + server::{ClientHello, ResolvesServerCert}, + sign::CertifiedKey, +}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[derive(Debug)] +struct FixedCertificate(Arc); + +impl ResolvesServerCert for FixedCertificate { + fn resolve(&self, _: ClientHello<'_>) -> Option> { + Some(self.0.clone()) + } +} + +#[tokio::test] +async fn tls12_requires_the_server_certificate_key_even_without_ca_validation() { + check_key_possession(&rustls::version::TLS12).await; +} + +#[tokio::test] +async fn tls13_requires_the_server_certificate_key_even_without_ca_validation() { + check_key_possession(&rustls::version::TLS13).await; +} + +async fn check_key_possession(version: &'static rustls::SupportedProtocolVersion) { + for correct_key in [true, false] { + let server_cert = TestCert::localhost().self_signed().unwrap(); + let signing_key = if correct_key { + server_cert.key_der() + } else { + TestCert::localhost().self_signed().unwrap().key_der() + }; + let ca = TestCa::new().unwrap(); + let client_cert = TestCert::localhost().signed_by(&ca).unwrap(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(ca.cert_key().cert.der().clone()).unwrap(); + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let client_verifier = rustls::server::WebPkiClientVerifier::builder_with_provider( + Arc::new(roots), + provider.clone(), + ) + .build() + .unwrap(); + let signing_key = provider + .key_provider + .load_private_key(PrivatePkcs8KeyDer::from(signing_key).into()) + .unwrap(); + // A resolver permits deliberately pairing a stolen certificate with + // another key, unlike with_single_cert which rejects it up front. + let pinned_cert = server_cert.cert_der(); + let certified = CertifiedKey::new(vec![server_cert.cert.der().clone()], signing_key); + let config = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[version]) + .unwrap() + .with_client_cert_verifier(client_verifier) + .with_cert_resolver(Arc::new(FixedCertificate(Arc::new(certified)))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let Ok(mut stream) = tokio_rustls::TlsAcceptor::from(Arc::new(config)) + .accept(stream) + .await + else { + return false; + }; + let mut request = [0; 4096]; + if !matches!(stream.read(&mut request).await, Ok(n) if n > 0) { + return false; + } + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nnull") + .await + .unwrap(); + true + }); + let client = RaClientConfig::builder() + .remote_uri(format!("https://{address}")) + .tls_skip_ca_verification(true) + .tls_client_cert(client_cert.cert_pem()) + .tls_client_key(client_cert.key_pem()) + .verify_server_attestation(false) + .cert_validator(Box::new(move |info| { + let info = info.ok_or_else(|| anyhow::anyhow!("missing server certificate"))?; + anyhow::ensure!(info.cert_der == pinned_cert, "server certificate changed"); + Ok(()) + })) + .build() + .into_client() + .unwrap(); + let result = client.request::<(), ()>("Probe", ()).await; + assert_eq!(result.is_ok(), correct_key, "{version:?}: {result:?}"); + assert_eq!(server.await.unwrap(), correct_key); + } +}