hashiverse_server_lib/transport/
https_transport_cert_refresher.rs

1//! # Let's Encrypt TLS certificate lifecycle manager
2//!
3//! Keeps the HTTPS transport's TLS certificate fresh without operator intervention.
4//! Uses `instant-acme` to talk to Let's Encrypt (production or staging, picked via
5//! [`hashiverse_lib::tools::config::USE_PRODUCTION_LETS_ENCRYPT`]) and
6//! TLS-ALPN-01 to solve domain-validation challenges inline on the same HTTPS port
7//! — no separate HTTP-01 listener needed.
8//!
9//! Two cert slots live side-by-side in `RwLock`s:
10//! - `base_cert` — the currently-serving cert.
11//! - `challenge_cert` — the short-lived self-signed (via `rcgen`) cert rustls serves
12//!   only when ACME is mid-challenge.
13//!
14//! Swapping slots is atomic, so a refresh never drops a live TLS handshake.
15//! Refresh cadence, retry-on-failure cadence, and renewal lead time all come from
16//! the `MILLIS_TO_WAIT_BETWEEN_CERT_*` constants in
17//! [`hashiverse_lib::tools::config`].
18
19use anyhow::Context;
20use hashiverse_lib::tools::config::USE_PRODUCTION_LETS_ENCRYPT;
21use std::path::Path;
22use hashiverse_lib::tools::time::{TimeMillis};
23use hashiverse_lib::tools::{config, tools};
24use instant_acme::{Account, AuthorizationStatus, ChallengeType, Identifier, LetsEncrypt, NewAccount, NewOrder, OrderStatus, RetryPolicy};
25use log::{info, trace, warn};
26use parking_lot::RwLock;
27use rcgen::{CertificateParams, SanType, generate_simple_self_signed};
28use rustls::server::{ClientHello, ResolvesServerCert};
29use rustls::sign::CertifiedKey;
30use rustls_pki_types::pem::PemObject;
31use rustls_pki_types::{CertificateDer, PrivateKeyDer};
32use std::fs;
33use std::net::IpAddr;
34use std::path::PathBuf;
35use std::str::FromStr;
36use std::sync::Arc;
37use tokio_util::sync::CancellationToken;
38use hashiverse_lib::tools::time_provider::time_provider::{RealTimeProvider, TimeProvider};
39
40pub const FILENAME_LAST_REFRESHED: &str = "last_refreshed";
41pub const FILENAME_CERT: &str = "cert.pem";
42pub const FILENAME_KEY: &str = "key.pem";
43
44#[derive(Debug)]
45pub struct HttpsTransportCertRefresher {
46    force_local_network: bool,
47    path_certs: PathBuf,
48    filename_cert: PathBuf,
49    filename_key: PathBuf,
50    filename_last_refreshed: PathBuf,
51
52    ip: String,
53    port: u16,
54
55    // The "real" certificate for your IP
56    pub base_cert: Arc<RwLock<Option<Arc<CertifiedKey>>>>,
57
58    // The temporary certificate used for the ACME challenge
59    pub challenge_cert: Arc<RwLock<Option<Arc<CertifiedKey>>>>,
60}
61
62impl HttpsTransportCertRefresher {
63    pub async fn refresh_cert(&self) -> anyhow::Result<()> {
64        trace!("refreshing certificate");
65
66        let directory_url = match USE_PRODUCTION_LETS_ENCRYPT {
67            true => LetsEncrypt::Production.url().to_owned(),
68            false => LetsEncrypt::Staging.url().to_owned(),
69        };
70
71        let (account, _credentials) = Account::builder()?
72            .create(
73                &NewAccount {
74                    contact: &[],
75                    terms_of_service_agreed: true,
76                    only_return_existing: false,
77                },
78                directory_url,
79                None,
80            )
81            .await?;
82
83        // info!("letsencrypt.credentials: {}", serde_json::to_string_pretty(&credentials)?);
84
85        let ip_addr = IpAddr::from_str(&self.ip)?;
86        let identifiers = [Identifier::Ip(ip_addr)];
87        let new_order = NewOrder::new(&identifiers).profile("shortlived");
88
89        let mut order = account.new_order(&new_order).await?;
90        let mut challenge_url = "".to_string();
91        // info!("order state: {:#?}", order.state());
92
93        // Fulfil any of the TlsAlpn01 authorizations we can (there should be only one)
94        {
95            let mut authorizations = order.authorizations();
96            while let Some(result) = authorizations.next().await {
97                let mut authz = result?;
98                match authz.status {
99                    AuthorizationStatus::Valid => continue,
100                    AuthorizationStatus::Pending => {}
101                    _ => {
102                        warn!("unexpected AuthorizationStatus {:?}", authz.status);
103                        continue;
104                    }
105                }
106
107                let mut challenge = authz.challenge(ChallengeType::TlsAlpn01).ok_or_else(|| anyhow::anyhow!("no tlsalpn01 challenge found"))?;
108                challenge_url = challenge.url.clone();
109
110                // Create the temporary challenge cert
111                {
112                    let key_auth_sha256 = challenge.key_authorization().digest();
113
114                    let mut certificate_params = CertificateParams::default();
115                    certificate_params.subject_alt_names = vec![SanType::IpAddress(ip_addr)];
116                    certificate_params.custom_extensions.push(rcgen::CustomExtension::new_acme_identifier(key_auth_sha256.as_ref()));
117
118                    let key_pair = rcgen::KeyPair::generate()?;
119                    let cert = certificate_params.self_signed(&key_pair)?;
120
121                    let cert_chain = vec![rustls_pki_types::CertificateDer::from(cert.der().to_vec())];
122                    let key_der = rustls_pki_types::PrivatePkcs8KeyDer::from(key_pair.serialize_der());
123                    let key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&rustls_pki_types::PrivateKeyDer::Pkcs8(key_der)).map_err(|_| anyhow::anyhow!("Unsupported key type"))?;
124
125                    let certified_key = CertifiedKey { cert: cert_chain, key, ocsp: None };
126
127                    *self.challenge_cert.write() = Some(Arc::new(certified_key));
128                }
129
130                // Tell letsencrypt that our challenge handler is ready for them
131                challenge.set_ready().await?;
132            }
133        }
134
135        //
136        trace!("challenge.url: {}", challenge_url);
137
138        // Wait for our request to be validated
139        let status = order.poll_ready(&RetryPolicy::default()).await?;
140        if status != OrderStatus::Ready {
141            anyhow::bail!("unexpected order status: {status:?}");
142        }
143
144        // Get the certificate
145        let private_key_pem = order.finalize().await?;
146        let cert_chain_pem = order.poll_certificate(&RetryPolicy::default()).await?;
147
148        // Write to disk
149        info!("writing new certificate to disk");
150        fs::create_dir_all(&self.path_certs)?;
151        fs::write(&self.filename_cert, cert_chain_pem)?;
152        write_private_key_file(&self.filename_key, private_key_pem.as_bytes())?;
153        fs::write(&self.filename_last_refreshed, challenge_url)?;
154
155        info!("refreshed certificate");
156
157        Ok(())
158    }
159
160    pub fn reload_certs(&self) -> anyhow::Result<()> {
161        trace!("reloading certificate");
162
163        let certified_key: CertifiedKey = {
164            let bytes_cert = fs::read(&self.filename_cert)?;
165            let bytes_key = fs::read(&self.filename_key)?;
166
167            // Return the CertifiedKey
168            let cert_chain = CertificateDer::pem_slice_iter(&bytes_cert).collect::<Result<Vec<_>, _>>().with_context(|| format!("parsing certificates in {}", self.filename_cert.display()))?;
169            // An empty/truncated cert file must not replace the certificate we are currently serving
170            if cert_chain.is_empty() {
171                anyhow::bail!("No certificates found in {}", self.filename_cert.display());
172            }
173            let key_der = PrivateKeyDer::from_pem_slice(&bytes_key).map_err(|e| anyhow::anyhow!("No private key found in {}: {}", self.filename_key.display(), e))?;
174            let key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key_der).map_err(|_| anyhow::anyhow!("Unsupported key type"))?;
175
176            CertifiedKey { cert: cert_chain, key, ocsp: None }
177        };
178
179        *self.base_cert.write() = Some(Arc::new(certified_key));
180
181        Ok(())
182    }
183
184    fn certs_last_refreshed(&self) -> anyhow::Result<TimeMillis> {
185        let certs_last_refreshed = fs::metadata(&self.filename_last_refreshed)
186            .map(|metadata| metadata.modified())
187            .unwrap_or_else(|_| Ok(std::time::SystemTime::UNIX_EPOCH))
188            .with_context(|| "checking last refreshed filename")?;
189
190        let certs_last_refreshed: TimeMillis = certs_last_refreshed.into();
191
192        Ok(certs_last_refreshed)
193    }
194
195    pub async fn process(&self, cancellation_token: CancellationToken) -> anyhow::Result<()> {
196        let time_provider = RealTimeProvider;
197
198        let mut certs_last_attempted = TimeMillis::zero();
199
200        loop {
201            if cancellation_token.is_cancelled() {
202                break;
203            }
204
205            let result: anyhow::Result<()> = try {
206                // Do nothing if we are forced local
207                if !self.force_local_network {
208                    // If we are in control of port 443, we can do the actual cert renewal from letsencrypt
209                    if 443u16 == self.port {
210                        // And do nothing if we have tried too recently
211                        let now_millis = time_provider.current_time_millis();
212                        if (now_millis - self.certs_last_refreshed()?) > config::MILLIS_TO_WAIT_BETWEEN_CERT_RENEWALS {
213                            if (now_millis - certs_last_attempted) > config::MILLIS_TO_WAIT_BETWEEN_CERT_RENEWAL_FAILURES {
214                                certs_last_attempted = now_millis;
215                                self.refresh_cert().await?;
216                            }
217                            else {
218                                trace!("we refreshed certs too recently to try again");
219                            }
220                        }
221                        else {
222                            trace!("we have a recent enough cert on disk");
223                        }
224                    }
225                    else {
226                        trace!("skipping cert refresh because port {} != 443", self.port);
227                    }
228                }
229                else {
230                    trace!("skipping cert refresh because force_local_network");
231                }
232
233                // Reload our certs from disk
234                self.reload_certs()?;
235            };
236
237            if let Err(e) = result {
238                warn!("error while refreshing certs: {}", e);
239            }
240
241            tools::cancellable_sleep_millis(&time_provider, config::MILLIS_TO_WAIT_BETWEEN_CERT_RENEWAL_CHECKS, &cancellation_token).await;
242        }
243
244        trace!("stopped HttpsTransportCertRefresher");
245
246        Ok(())
247    }
248}
249
250/// Write a private key file with owner-read-only permissions (0600) on Unix.
251/// Falls back to a plain write on non-Unix platforms.
252fn write_private_key_file(path: &Path, contents: &[u8]) -> anyhow::Result<()> {
253    #[cfg(unix)]
254    {
255        use std::io::Write;
256        use std::os::unix::fs::OpenOptionsExt;
257        let mut file = fs::OpenOptions::new()
258            .write(true)
259            .create(true)
260            .truncate(true)
261            .mode(0o600)
262            .open(path)?;
263        file.write_all(contents)?;
264    }
265    #[cfg(not(unix))]
266    {
267        fs::write(path, contents)?;
268    }
269    Ok(())
270}
271
272impl HttpsTransportCertRefresher {
273    pub fn new(path_certs: PathBuf, ip: String, port: u16, force_local_network: bool) -> anyhow::Result<Self> {
274        let filename_cert = path_certs.join(FILENAME_CERT);
275        let filename_key = path_certs.join(FILENAME_KEY);
276        let filename_last_refreshed = path_certs.join(FILENAME_LAST_REFRESHED);
277
278        // Make sure we start with at least some baseline certificates.
279        // Self-signed certs are generated whenever none exist on disk — including force_local_network
280        // (test/local) mode.  For production, Let's Encrypt replaces them via process().
281        if !fs::exists(&filename_cert)? || !fs::exists(&filename_key)? {
282            info!("generating self-signed certs");
283            let subject_alt_names = vec![ip.clone()];
284
285            let certified_key = generate_simple_self_signed(subject_alt_names)?;
286            fs::create_dir_all(&path_certs)?;
287            fs::write(&filename_cert, certified_key.cert.pem())?;
288            write_private_key_file(&filename_key, certified_key.signing_key.serialize_pem().as_bytes())?;
289        }
290
291        // Initially our certs are empty
292        let base_cert = Arc::new(RwLock::new(None));
293        let challenge_cert = Arc::new(RwLock::new(None));
294
295        Ok(Self {
296            force_local_network,
297            path_certs,
298            filename_cert,
299            filename_key,
300            filename_last_refreshed,
301            ip,
302            port,
303            base_cert,
304            challenge_cert,
305        })
306    }
307}
308
309impl ResolvesServerCert for HttpsTransportCertRefresher {
310    fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
311        // Check if the client (the CA) is specifically asking for the ACME protocol
312        let is_acme = client_hello.alpn().map(|mut iter| iter.any(|proto| proto == b"acme-tls/1")).unwrap_or(false);
313
314        if is_acme {
315            // Return the temporary challenge cert generated from instant-acme tokens
316            info!("we have an acme challenge");
317            self.challenge_cert.read().clone()
318        }
319        else {
320            // Return the standard certificate for your IP
321            self.base_cert.read().clone()
322        }
323    }
324}
325
326#[cfg(test)]
327mod tests {
328    use super::*;
329    use hashiverse_lib::tools::tools::get_temp_dir;
330
331    const TEST_IP: &str = "127.0.0.1";
332
333    /// Writes a fresh rcgen self-signed cert + key into `path_certs` (before `new()` runs, so `new()` keeps them)
334    /// and returns the DER of the cert so tests can compare against an independently-known value.
335    fn write_self_signed_pem_files(path_certs: &Path, extra_chain_certificate_count: usize) -> anyhow::Result<Vec<Vec<u8>>> {
336        fs::create_dir_all(path_certs)?;
337        let mut expected_chain_ders = Vec::new();
338        let mut cert_chain_pem = String::new();
339
340        let leaf_certified_key = generate_simple_self_signed(vec![TEST_IP.to_string()])?;
341        expected_chain_ders.push(leaf_certified_key.cert.der().to_vec());
342        cert_chain_pem.push_str(&leaf_certified_key.cert.pem());
343
344        for _ in 0..extra_chain_certificate_count {
345            let intermediate_certified_key = generate_simple_self_signed(vec![TEST_IP.to_string()])?;
346            expected_chain_ders.push(intermediate_certified_key.cert.der().to_vec());
347            cert_chain_pem.push_str(&intermediate_certified_key.cert.pem());
348        }
349
350        fs::write(path_certs.join(FILENAME_CERT), cert_chain_pem)?;
351        write_private_key_file(&path_certs.join(FILENAME_KEY), leaf_certified_key.signing_key.serialize_pem().as_bytes())?;
352        Ok(expected_chain_ders)
353    }
354
355    fn loaded_chain_ders(https_transport_cert_refresher: &HttpsTransportCertRefresher) -> Option<Vec<Vec<u8>>> {
356        https_transport_cert_refresher.base_cert.read().as_ref().map(|certified_key| certified_key.cert.iter().map(|certificate_der| certificate_der.as_ref().to_vec()).collect())
357    }
358
359    #[test]
360    fn new_generates_self_signed_pems_that_reload_certs_accepts() -> anyhow::Result<()> {
361        let (_temp_dir_handle, temp_dir_path) = get_temp_dir()?;
362        let path_certs = PathBuf::from(&temp_dir_path).join("certs");
363
364        let https_transport_cert_refresher = HttpsTransportCertRefresher::new(path_certs.clone(), TEST_IP.to_string(), 0, true)?;
365        assert!(path_certs.join(FILENAME_CERT).exists());
366        assert!(path_certs.join(FILENAME_KEY).exists());
367        assert!(https_transport_cert_refresher.base_cert.read().is_none());
368
369        https_transport_cert_refresher.reload_certs()?;
370        let loaded_chain_ders = loaded_chain_ders(&https_transport_cert_refresher).expect("base_cert populated after reload");
371        assert_eq!(loaded_chain_ders.len(), 1);
372        Ok(())
373    }
374
375    #[test]
376    fn reload_certs_loads_full_chain_in_order() -> anyhow::Result<()> {
377        let (_temp_dir_handle, temp_dir_path) = get_temp_dir()?;
378        let path_certs = PathBuf::from(&temp_dir_path).join("certs");
379        let expected_chain_ders = write_self_signed_pem_files(&path_certs, 1)?;
380
381        let https_transport_cert_refresher = HttpsTransportCertRefresher::new(path_certs, TEST_IP.to_string(), 0, true)?;
382        https_transport_cert_refresher.reload_certs()?;
383
384        assert_eq!(loaded_chain_ders(&https_transport_cert_refresher), Some(expected_chain_ders));
385        Ok(())
386    }
387
388    #[test]
389    fn reload_certs_errors_when_key_file_has_no_private_key() -> anyhow::Result<()> {
390        let (_temp_dir_handle, temp_dir_path) = get_temp_dir()?;
391        let path_certs = PathBuf::from(&temp_dir_path).join("certs");
392        write_self_signed_pem_files(&path_certs, 0)?;
393
394        // A key file holding only a certificate block must be rejected, not silently accepted
395        let cert_pem = fs::read(path_certs.join(FILENAME_CERT))?;
396        fs::write(path_certs.join(FILENAME_KEY), cert_pem)?;
397
398        let https_transport_cert_refresher = HttpsTransportCertRefresher::new(path_certs, TEST_IP.to_string(), 0, true)?;
399        assert!(https_transport_cert_refresher.reload_certs().is_err());
400        assert!(https_transport_cert_refresher.base_cert.read().is_none());
401        Ok(())
402    }
403
404    #[test]
405    fn reload_certs_with_empty_cert_file_keeps_previous_certificate() -> anyhow::Result<()> {
406        let (_temp_dir_handle, temp_dir_path) = get_temp_dir()?;
407        let path_certs = PathBuf::from(&temp_dir_path).join("certs");
408        let expected_chain_ders = write_self_signed_pem_files(&path_certs, 0)?;
409
410        let https_transport_cert_refresher = HttpsTransportCertRefresher::new(path_certs.clone(), TEST_IP.to_string(), 0, true)?;
411        https_transport_cert_refresher.reload_certs()?;
412        assert_eq!(loaded_chain_ders(&https_transport_cert_refresher), Some(expected_chain_ders.clone()));
413
414        // e.g. a truncated write during refresh: the reload must fail and keep serving the previous certificate
415        fs::write(path_certs.join(FILENAME_CERT), b"")?;
416        assert!(https_transport_cert_refresher.reload_certs().is_err());
417        assert_eq!(loaded_chain_ders(&https_transport_cert_refresher), Some(expected_chain_ders));
418        Ok(())
419    }
420
421    #[test]
422    fn reload_certs_errors_when_files_missing() -> anyhow::Result<()> {
423        let (_temp_dir_handle, temp_dir_path) = get_temp_dir()?;
424        let path_certs = PathBuf::from(&temp_dir_path).join("certs");
425
426        let https_transport_cert_refresher = HttpsTransportCertRefresher::new(path_certs.clone(), TEST_IP.to_string(), 0, true)?;
427        fs::remove_file(path_certs.join(FILENAME_KEY))?;
428        assert!(https_transport_cert_refresher.reload_certs().is_err());
429        assert!(https_transport_cert_refresher.base_cert.read().is_none());
430        Ok(())
431    }
432}
hashiverse_server_lib/transport/https_transport_cert_refresher.rs

hashiverse_server_lib/transport/
https_transport_cert_refresher.rs
