diff --git a/desktop/src/app.rs b/desktop/src/app.rs index a69b9cacf90..96cfbb1b565 100644 --- a/desktop/src/app.rs +++ b/desktop/src/app.rs @@ -25,7 +25,7 @@ use crate::render::{RenderError, RenderState}; use crate::ui::{InputEvent, UiCommand, UiInstance}; use crate::window::Window; use crate::wrapper::messages::{DesktopFrontendMessage, DesktopWrapperMessage, IngestAction, Preferences}; -use crate::wrapper::{DesktopWrapper, MmapResourceStorage, NodeGraphExecutionResult, WgpuContext, serialize_frontend_messages}; +use crate::wrapper::{DesktopWrapper, FolderStore, MmapResourceStorage, NodeGraphExecutionResult, WgpuContext, serialize_frontend_messages}; pub(crate) struct App { render_state: Option, @@ -99,7 +99,13 @@ impl App { let wake = Arc::new(move || { wake_scheduler.schedule(AppEvent::DesktopWrapperMessage(DesktopWrapperMessage::Wake)); }); - let desktop_wrapper = DesktopWrapper::new(rand::rng().random(), Arc::new(resource_storage), dirs::app_autosave_documents_dir(), wgpu_context.clone(), wake); + let desktop_wrapper = DesktopWrapper::new( + rand::rng().random(), + Arc::new(resource_storage), + Arc::new(FolderStore::new(dirs::app_autosave_documents_dir())), + wgpu_context.clone(), + wake, + ); Self { render_state: None, @@ -307,19 +313,6 @@ impl App { DesktopFrontendMessage::PersistenceReadState => { responses.push(DesktopWrapperMessage::LoadPersistedState { state: persist::read_state() }); } - DesktopFrontendMessage::PersistenceReadDocument { id } => { - if let Some(document) = persist::read_document_content(&id) { - responses.push(DesktopWrapperMessage::LoadDocumentContent { id, document }); - } else { - tracing::error!("Failed to read document content for {id:?}"); - } - } - DesktopFrontendMessage::PersistenceWriteDocument { id, document_serialized_content } => { - persist::write_document_content(id, document_serialized_content); - } - DesktopFrontendMessage::PersistenceDeleteDocument { id } => { - persist::delete_document(&id); - } DesktopFrontendMessage::PersistenceWritePreferences { preferences } => { preferences::write(preferences); } diff --git a/desktop/src/persist.rs b/desktop/src/persist.rs index 7ac387a33f0..465a13e1d84 100644 --- a/desktop/src/persist.rs +++ b/desktop/src/persist.rs @@ -1,6 +1,9 @@ -use crate::wrapper::messages::{DocumentId, PersistedState}; +use crate::wrapper::messages::PersistedState; pub(crate) fn read_state() -> PersistedState { + if let Err(error) = migrate_documents() { + tracing::error!("Autosave migration failed: {error}"); + } let path = state_file_path(); let data = match std::fs::read_to_string(&path) { Ok(d) => d, @@ -13,16 +16,13 @@ pub(crate) fn read_state() -> PersistedState { return PersistedState::default(); } }; - let loaded = match ron::from_str(&data) { + match ron::from_str(&data) { Ok(d) => d, Err(e) => { tracing::error!("Failed to deserialize persistent data: {e}"); - return PersistedState::default(); + PersistedState::default() } - }; - - garbage_collect_document_files(&loaded); - loaded + } } pub(crate) fn write_state(state: PersistedState) { @@ -37,57 +37,55 @@ pub(crate) fn write_state(state: PersistedState) { if let Err(e) = std::fs::write(state_file_path(), data) { tracing::error!("Failed to write persistent data to disk: {e}"); } - garbage_collect_document_files(state); -} - -pub(crate) fn write_document_content(id: DocumentId, document_content: String) { - if let Err(e) = std::fs::write(document_content_path(&id), document_content) { - tracing::error!("Failed to write document {id:?} to disk: {e}"); - } } -pub(crate) fn read_document_content(id: &DocumentId) -> Option { - std::fs::read_to_string(document_content_path(id)).ok() -} - -pub(crate) fn delete_document(id: &DocumentId) { - if let Err(e) = std::fs::remove_file(document_content_path(id)) { - tracing::error!("Failed to delete document {id:?} from disk: {e}"); - } +fn state_file_path() -> std::path::PathBuf { + let mut path = crate::dirs::app_data_dir(); + path.push(crate::consts::APP_STATE_FILE_NAME); + path } -fn garbage_collect_document_files(state: &PersistedState) { - let valid_paths: std::collections::HashSet<_> = state.documents.iter().map(|doc| document_content_path(&doc.id)).collect(); - - let directory = crate::dirs::app_autosave_documents_dir(); - let entries = match std::fs::read_dir(&directory) { +// TODO: Remove this migration code +fn migrate_documents() -> std::io::Result<()> { + let root = crate::dirs::app_autosave_documents_dir(); + let entries = match std::fs::read_dir(&root) { Ok(entries) => entries, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return, - Err(e) => { - tracing::error!("Failed to read autosave documents directory: {e}"); - return; - } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), }; - - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() - && !valid_paths.contains(&path) - && let Err(e) = std::fs::remove_file(&path) - { - tracing::error!("Failed to remove orphaned document file {path:?}: {e}"); + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + tracing::error!("Skipping an unreadable autosave entry: {error}"); + continue; + } + }; + if let Err(error) = migrate_document(&root, &entry) { + tracing::error!("Autosave migration of {} failed: {error}", entry.path().display()); } } + Ok(()) } -fn state_file_path() -> std::path::PathBuf { - let mut path = crate::dirs::app_data_dir(); - path.push(crate::consts::APP_STATE_FILE_NAME); - path -} - -fn document_content_path(id: &DocumentId) -> std::path::PathBuf { - let mut path = crate::dirs::app_autosave_documents_dir(); - path.push(format!("{:x}.{}", id.0, graphite_desktop_wrapper::FILE_EXTENSION)); - path +fn migrate_document(root: &std::path::Path, entry: &std::fs::DirEntry) -> std::io::Result<()> { + let path = entry.path(); + if !entry.file_type()?.is_file() || path.extension().and_then(|s| s.to_str()) != Some(graphite_desktop_wrapper::FILE_EXTENSION) { + return Ok(()); + } + let Some(id) = path.file_stem().and_then(|s| s.to_str()).and_then(|s| u64::from_str_radix(s, 16).ok()) else { + return Ok(()); + }; + let destination = root.join(format!("{id:016x}")); + if !destination.join("legacy.graphite").exists() && !destination.join("manifest.json").exists() { + std::fs::create_dir_all(&destination)?; + let staging = destination.join("legacy.graphite.migrating"); + std::fs::copy(&path, &staging)?; + std::fs::rename(staging, destination.join("legacy.graphite"))?; + } + let backup = path.with_extension(format!("{}.migrated", graphite_desktop_wrapper::FILE_EXTENSION)); + if backup.exists() { + return Ok(()); + } + std::fs::rename(path, backup) } diff --git a/desktop/wrapper/src/handle_desktop_wrapper_message.rs b/desktop/wrapper/src/handle_desktop_wrapper_message.rs index 7a4dafbe0af..df3237623af 100644 --- a/desktop/wrapper/src/handle_desktop_wrapper_message.rs +++ b/desktop/wrapper/src/handle_desktop_wrapper_message.rs @@ -44,10 +44,6 @@ pub(super) fn handle_desktop_wrapper_message(dispatcher: &mut DesktopWrapperMess let message = FrontendMessage::UpdateFullscreen { fullscreen }; dispatcher.queue_editor_message(message); } - DesktopWrapperMessage::LoadDocumentContent { id, document } => { - let message = PersistentStateMessage::LoadDocument { document_id: id, document }; - dispatcher.queue_editor_message(message); - } DesktopWrapperMessage::LoadPersistedState { state } => { let message = PersistentStateMessage::LoadState { state }; dispatcher.queue_editor_message(message); diff --git a/desktop/wrapper/src/intercept_frontend_message.rs b/desktop/wrapper/src/intercept_frontend_message.rs index f5c71f8bc47..27e0a6892ba 100644 --- a/desktop/wrapper/src/intercept_frontend_message.rs +++ b/desktop/wrapper/src/intercept_frontend_message.rs @@ -69,18 +69,6 @@ pub(super) fn intercept_frontend_message(dispatcher: &mut DesktopWrapperMessageD FrontendMessage::TriggerPersistenceWriteState { state } => { dispatcher.respond(DesktopFrontendMessage::PersistenceWriteState { state }); } - FrontendMessage::TriggerPersistenceReadDocument { document_id } => { - dispatcher.respond(DesktopFrontendMessage::PersistenceReadDocument { id: document_id }); - } - FrontendMessage::TriggerPersistenceDeleteDocument { document_id } => { - dispatcher.respond(DesktopFrontendMessage::PersistenceDeleteDocument { id: document_id }); - } - FrontendMessage::TriggerPersistenceWriteDocument { document_id, document } => { - dispatcher.respond(DesktopFrontendMessage::PersistenceWriteDocument { - id: document_id, - document_serialized_content: document, - }); - } FrontendMessage::TriggerOpenLaunchDocuments => { dispatcher.respond(DesktopFrontendMessage::OpenLaunchDocuments); } diff --git a/desktop/wrapper/src/lib.rs b/desktop/wrapper/src/lib.rs index ebe6458c354..e84e5e72888 100644 --- a/desktop/wrapper/src/lib.rs +++ b/desktop/wrapper/src/lib.rs @@ -10,6 +10,7 @@ use std::sync::Arc; pub use graph_craft::application_io::resource::MmapResourceStorage; pub use graphite_editor::consts::{DOUBLE_CLICK_MILLISECONDS, FILE_EXTENSION}; +pub use graphite_editor::document_store::{DocumentStore, FolderStore}; pub use wgpu_executor::Texture; pub use wgpu_executor::WgpuBackends; pub use wgpu_executor::WgpuContext; @@ -32,7 +33,7 @@ pub struct DesktopWrapper { } impl DesktopWrapper { - pub fn new(uuid_random_seed: u64, resource_storage: Arc, working_copy_root: std::path::PathBuf, wgpu_context: WgpuContext, schedule_wake: Wake) -> Self { + pub fn new(uuid_random_seed: u64, resource_storage: Arc, document_store: Arc, wgpu_context: WgpuContext, schedule_wake: Wake) -> Self { #[cfg(target_os = "windows")] let host = Host::Windows; #[cfg(target_os = "macos")] @@ -43,7 +44,7 @@ impl DesktopWrapper { let application_io = PlatformApplicationIo::new_with_context(wgpu_context); Self { - editor: Editor::new(env, uuid_random_seed, resource_storage, Some(working_copy_root), application_io, schedule_wake), + editor: Editor::new(env, uuid_random_seed, resource_storage, document_store, application_io, schedule_wake), } } diff --git a/desktop/wrapper/src/messages.rs b/desktop/wrapper/src/messages.rs index 2bce1100e80..b3663ea87eb 100644 --- a/desktop/wrapper/src/messages.rs +++ b/desktop/wrapper/src/messages.rs @@ -44,13 +44,6 @@ pub enum DesktopFrontendMessage { enabled: bool, }, UpdateOverlays(vello::Scene), - PersistenceWriteDocument { - id: DocumentId, - document_serialized_content: String, - }, - PersistenceDeleteDocument { - id: DocumentId, - }, PersistenceWritePreferences { preferences: Preferences, }, @@ -59,9 +52,6 @@ pub enum DesktopFrontendMessage { state: PersistedState, }, PersistenceReadState, - PersistenceReadDocument { - id: DocumentId, - }, UpdateMenu { entries: Vec, }, @@ -92,7 +82,6 @@ pub enum DesktopWrapperMessage { PollNodeGraphEvaluation, UpdateMaximized { maximized: bool }, UpdateFullscreen { fullscreen: bool }, - LoadDocumentContent { id: DocumentId, document: String }, LoadPersistedState { state: PersistedState }, LoadPreferences { preferences: Preferences }, MenuEvent { id: String }, diff --git a/document/container/Cargo.toml b/document/container/Cargo.toml index 79080e4d3e4..c9201b14d70 100644 --- a/document/container/Cargo.toml +++ b/document/container/Cargo.toml @@ -34,6 +34,7 @@ web-sys = { workspace = true, features = [ "FileSystemFileHandle", "FileSystemGetFileOptions", "FileSystemGetDirectoryOptions", + "FileSystemRemoveOptions", "FileSystemHandle", "FileSystemHandleKind", "FileSystemWritableFileStream", diff --git a/document/container/src/backends/folder.rs b/document/container/src/backends/folder.rs index c7327e976f7..27f5df23b67 100644 --- a/document/container/src/backends/folder.rs +++ b/document/container/src/backends/folder.rs @@ -6,6 +6,7 @@ use std::fs::{self, OpenOptions}; use std::io::Write; use std::path::{Path, PathBuf}; +#[derive(Clone)] pub struct FolderBackend { root: PathBuf, } diff --git a/document/container/src/backends/memory.rs b/document/container/src/backends/memory.rs index 0509536798c..8582bac980a 100644 --- a/document/container/src/backends/memory.rs +++ b/document/container/src/backends/memory.rs @@ -2,11 +2,11 @@ use crate::{ByteHolder, Container, ContainerError, Result, validate_path, validate_prefix, with_trailing_slash}; use std::collections::{HashMap, HashSet}; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; -#[derive(Default)] +#[derive(Clone, Default)] pub struct MemoryBackend { - files: Mutex>>, + files: Arc>>>, } impl MemoryBackend { diff --git a/document/container/src/backends/opfs.rs b/document/container/src/backends/opfs.rs index dd25b2f6057..b5a8a8b8861 100644 --- a/document/container/src/backends/opfs.rs +++ b/document/container/src/backends/opfs.rs @@ -3,6 +3,7 @@ use crate::{AsyncContainer, ByteHolder, ContainerError, Result, validate_path, validate_prefix, with_trailing_slash}; use futures::channel::oneshot; use js_sys::Uint8Array; +use std::cell::Cell; use std::collections::{HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use wasm_bindgen::JsCast; @@ -40,6 +41,33 @@ struct Inner { worker_active: bool, } +thread_local! { + static LAST_WRITE_FAILURE_ALERT: Cell> = const { Cell::new(None) }; +} + +fn report_write_error(operation: &str, path: &str, error: &JsValue) { + if is_not_found(error) { + log::warn!("OPFS {operation} on {path} dropped: the container no longer exists"); + return; + } + log::error!("OPFS {operation} failure on {path}:\n{error:?}"); + let now = js_sys::Date::now(); + // Debounce alerts to avoid spamming the user + if LAST_WRITE_FAILURE_ALERT.get().is_some_and(|last| now - last < 60. * 1000.) { + return; + } + LAST_WRITE_FAILURE_ALERT.set(Some(now)); + if let Some(window) = web_sys::window() { + let message = format!( + "Autosave is not working. Save your documents manually to avoid losing changes.\n\n\ + Please report it at https://github.com/GraphiteEditor/Graphite/issues/new and include this error:\n\n\ + {operation} {path}: {error:?}" + ); + let _ = window.alert_with_message(&message); + } +} + +#[derive(Clone)] pub struct OpfsBackend { inner: Arc>, } @@ -77,8 +105,6 @@ impl OpfsBackend { guard.queue.push_back(Mutation::Barrier(sender)); kick_worker(&self.inner, &mut guard); } - // The sender is only dropped without sending if the worker is torn down mid-drain; either way - // there is nothing left to wait for, so a receive error is treated as "already flushed". let _ = receiver.await; } } @@ -239,12 +265,12 @@ async fn drain_queue(inner: Arc>) { match mutation { Mutation::Write { path, bytes } => { if let Err(error) = write_file(&directory, &path, &bytes).await { - log::error!("OPFS background write for {path} failed: {error:?}"); + report_write_error("write", &path, &error); } } Mutation::Append { path, bytes } => { if let Err(error) = append_file(&directory, &path, &bytes).await { - log::error!("OPFS background append for {path} failed: {error:?}"); + report_write_error("append", &path, &error); } } Mutation::Delete { path } => { @@ -252,7 +278,7 @@ async fn drain_queue(inner: Arc>) { if let Err(error) = remove_file(&directory, &path).await && !is_not_found(&error) { - log::error!("OPFS background delete for {path} failed: {error:?}"); + report_write_error("delete", &path, &error); } } // A receive error on the waiter side just means the reader gave up; nothing to apply. @@ -263,16 +289,11 @@ async fn drain_queue(inner: Arc>) { } } -fn js_err(error: JsValue) -> ContainerError { - ContainerError::Backend(format!("{error:?}")) -} - /// Resolve `directory_path` (a `/`-separated relative path) under the OPFS root, creating each /// segment. OPFS rejects directory names containing `/`, so a multi-segment path like /// `documents/` must be descended one segment at a time rather than passed whole. async fn open_directory(directory_path: &str) -> std::result::Result { - let storage = web_sys::window().ok_or_else(|| JsValue::from_str("no window"))?.navigator().storage(); - let mut current: FileSystemDirectoryHandle = JsFuture::from(storage.get_directory()).await?.dyn_into()?; + let mut current = opfs_root().await?; for segment in directory_path.split('/').filter(|segment| !segment.is_empty()) { let options = FileSystemGetDirectoryOptions::new(); @@ -283,6 +304,37 @@ async fn open_directory(directory_path: &str) -> std::result::Result Result> { + crate::validate_prefix(directory_path)?; + let root = opfs_root().await.map_err(js_err)?; + list_entries(&root, directory_path, EntryKind::Directory).await.map_err(|error| list_error(directory_path, error)) +} + +/// Remove `directory_path` recursively. +pub async fn remove_directory(directory_path: &str) -> Result<()> { + validate_path(directory_path)?; + let root = opfs_root().await.map_err(js_err)?; + let removed = async { + let (parent, name) = descend(&root, directory_path, false).await?; + let options = web_sys::FileSystemRemoveOptions::new(); + options.set_recursive(true); + JsFuture::from(parent.remove_entry_with_options(name, &options)).await + } + .await; + + match removed { + Ok(_) => Ok(()), + Err(error) if is_not_found(&error) => Ok(()), + Err(error) => Err(js_err(error)), + } +} + +async fn opfs_root() -> std::result::Result { + let storage = web_sys::window().ok_or_else(|| JsValue::from_str("no window"))?.navigator().storage(); + JsFuture::from(storage.get_directory()).await?.dyn_into() +} + /// Descend the `/`-separated path against `root` and return the directory handle plus the final segment. async fn descend<'a>(root: &FileSystemDirectoryHandle, relative: &'a str, create_dirs: bool) -> std::result::Result<(FileSystemDirectoryHandle, &'a str), JsValue> { let mut current = root.clone(); @@ -473,3 +525,7 @@ fn list_error(prefix: &str, error: JsValue) -> ContainerError { js_err(error) } } + +fn js_err(error: JsValue) -> ContainerError { + ContainerError::Backend(format!("{error:?}")) +} diff --git a/document/container/src/lib.rs b/document/container/src/lib.rs index 49aa83a1e47..48c5482fa5b 100644 --- a/document/container/src/lib.rs +++ b/document/container/src/lib.rs @@ -12,6 +12,7 @@ pub mod archive; pub mod backends; +pub mod store; pub enum ByteHolder { /// Bytes synthesized in memory (decompressed from an archive, produced by serialization). @@ -350,6 +351,7 @@ impl AsyncContainer for C { /// `AsyncContainer::read` returns `impl Future`, so `dyn AsyncContainer` is not object-safe. /// `AnyContainer` is the workaround: `Gdd` holds one of these by value, and the `AsyncContainer` /// impl forwards to the active variant. +#[derive(Clone)] pub enum AnyContainer { Memory(backends::memory::MemoryBackend), #[cfg(not(target_family = "wasm"))] diff --git a/document/container/src/store.rs b/document/container/src/store.rs new file mode 100644 index 00000000000..48e42e22eec --- /dev/null +++ b/document/container/src/store.rs @@ -0,0 +1,145 @@ +use crate::backends::memory::MemoryBackend; +use crate::{AnyContainer, Result}; +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::Mutex; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DocumentKey(pub u64); + +#[cfg(not(target_family = "wasm"))] +pub type StoreFuture<'a, T> = Pin> + Send + 'a>>; +#[cfg(target_family = "wasm")] +pub type StoreFuture<'a, T> = Pin> + 'a>>; + +pub trait DocumentStore: Send + Sync { + fn list(&self) -> StoreFuture<'_, Vec>; + fn open(&self, key: DocumentKey) -> StoreFuture<'_, AnyContainer>; + fn remove(&self, key: DocumentKey) -> StoreFuture<'_, ()>; +} + +fn directory_name(key: DocumentKey) -> String { + format!("{:016x}", key.0) +} + +fn parse_directory_name(name: &str) -> Option { + (name.len() == 16 && name.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then(|| u64::from_str_radix(name, 16).ok().map(DocumentKey)) + .flatten() +} + +#[derive(Default)] +pub struct MemoryStore { + documents: Mutex>, +} + +impl DocumentStore for MemoryStore { + fn list(&self) -> StoreFuture<'_, Vec> { + let mut keys: Vec<_> = self.documents.lock().unwrap().keys().copied().collect(); + keys.sort(); + Box::pin(async move { Ok(keys) }) + } + + fn open(&self, key: DocumentKey) -> StoreFuture<'_, AnyContainer> { + let backend = self.documents.lock().unwrap().entry(key).or_default().clone(); + Box::pin(async move { Ok(AnyContainer::Memory(backend)) }) + } + + fn remove(&self, key: DocumentKey) -> StoreFuture<'_, ()> { + self.documents.lock().unwrap().remove(&key); + Box::pin(async { Ok(()) }) + } +} + +#[cfg(not(target_family = "wasm"))] +pub struct FolderStore { + root: std::path::PathBuf, +} + +#[cfg(not(target_family = "wasm"))] +impl FolderStore { + pub fn new(root: impl Into) -> Self { + Self { root: root.into() } + } +} + +#[cfg(not(target_family = "wasm"))] +impl DocumentStore for FolderStore { + fn list(&self) -> StoreFuture<'_, Vec> { + Box::pin(async { + let entries = match std::fs::read_dir(&self.root) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error.into()), + }; + + let mut keys: Vec<_> = entries + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_type().is_ok_and(|file_type| file_type.is_dir())) + .filter_map(|entry| parse_directory_name(&entry.file_name().to_string_lossy())) + .collect(); + keys.sort(); + Ok(keys) + }) + } + + fn open(&self, key: DocumentKey) -> StoreFuture<'_, AnyContainer> { + Box::pin(async move { + let backend = crate::backends::folder::FolderBackend::create(self.root.join(directory_name(key)))?; + Ok(AnyContainer::Folder(backend)) + }) + } + + fn remove(&self, key: DocumentKey) -> StoreFuture<'_, ()> { + Box::pin(async move { + match std::fs::remove_dir_all(self.root.join(directory_name(key))) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + }) + } +} + +#[cfg(target_family = "wasm")] +pub struct OpfsStore { + root: String, +} + +#[cfg(target_family = "wasm")] +impl OpfsStore { + pub fn new(root: impl Into) -> Self { + Self { + root: root.into().trim_matches('/').to_string(), + } + } + + fn document_path(&self, key: DocumentKey) -> String { + let name = directory_name(key); + if self.root.is_empty() { name } else { format!("{}/{name}", self.root) } + } +} + +#[cfg(target_family = "wasm")] +impl DocumentStore for OpfsStore { + fn list(&self) -> StoreFuture<'_, Vec> { + Box::pin(async { + let paths = crate::backends::opfs::list_directories(&self.root).await?; + let mut keys: Vec<_> = paths.iter().filter_map(|path| parse_directory_name(path.rsplit('/').next()?)).collect(); + keys.sort(); + Ok(keys) + }) + } + + fn open(&self, key: DocumentKey) -> StoreFuture<'_, AnyContainer> { + Box::pin(async move { + let backend = crate::backends::opfs::OpfsBackend::open(&self.document_path(key)).await?; + Ok(AnyContainer::Opfs(backend)) + }) + } + + fn remove(&self, key: DocumentKey) -> StoreFuture<'_, ()> { + Box::pin(async move { crate::backends::opfs::remove_directory(&self.document_path(key)).await }) + } +} diff --git a/document/container/tests/round_trip.rs b/document/container/tests/round_trip.rs index 2aa72b4c178..a6218621662 100644 --- a/document/container/tests/round_trip.rs +++ b/document/container/tests/round_trip.rs @@ -176,3 +176,15 @@ fn folder_backend_write_sized_fills_via_mmap() { let read_back = backend.read("resources/sized").unwrap(); assert_eq!(read_back.as_slice(), payload); } + +#[test] +fn memory_backend_clones_share_the_store() { + let backend = MemoryBackend::new(); + let clone = backend.clone(); + + backend.write("file", b"through the original").unwrap(); + assert_eq!(clone.read("file").unwrap().as_slice(), b"through the original"); + + clone.remove("file").unwrap(); + assert!(!backend.exists("file")); +} diff --git a/document/format/src/error.rs b/document/format/src/error.rs index b4d1d1811ea..d91574fa801 100644 --- a/document/format/src/error.rs +++ b/document/format/src/error.rs @@ -35,6 +35,9 @@ pub enum Error { #[cfg(feature = "conversion")] #[error("commit error: {0}")] Commit(#[from] CommitError), + /// The container has no manifest, cannot be opened. + #[error("container has no manifest")] + MissingManifest, /// The manifest's `format` field is not the `.gdd` magic, so this is not a `.gdd` document. #[error("not a .gdd document (manifest format = {found:?}, expected {expected:?})")] WrongFormat { found: String, expected: &'static str }, diff --git a/document/format/src/lib.rs b/document/format/src/lib.rs index 63f49b74ccf..bcc66afa390 100644 --- a/document/format/src/lib.rs +++ b/document/format/src/lib.rs @@ -6,7 +6,6 @@ //! //! See the "On-disk container" section of `node-graph/rfcs/document-format.md` for the format spec. -use std::sync::Arc; // `Path` and `FolderBackend` are only used by the native-only path-based open/create, so they're // gated off wasm to avoid unused-import warnings. #[cfg(not(target_family = "wasm"))] @@ -14,7 +13,7 @@ use std::path::Path; #[cfg(not(target_family = "wasm"))] use document_container::backends::folder::FolderBackend; -use document_container::{AnyContainer, AsyncContainer, ByteHolder, ContainerError}; +use document_container::{AnyContainer, AsyncContainer, ByteHolder}; #[cfg(feature = "conversion")] use document_graph_storage::{CommitError, NodeMetadataSource}; use document_graph_storage::{Delta, HotOp, PeerId, Registry, Session}; @@ -64,15 +63,15 @@ pub const DEFAULT_HOT_LOG_CODEC: Codec = Codec::MessagePackFrames; /// /// The per-edit persist path (`commit_from_runtime`, `apply_hot_op`, `retire`) is synchronous and /// read-free: the manifest is cached in memory (so payload codecs need no disk read), and writes go -/// through the container's sync write surface. Only `open` / `create` / `export` are async, since they +/// through the container's sync write surface. Only `open` / `export` are async, since they /// read. -/// `Clone` shares the working-copy container (`Arc`) so a cloned handle reads and writes +/// `Clone` shares the working-copy container (`AnyContainer` is a handle) so a cloned handle reads and writes /// the *same* on-disk/OPFS working copy — including any writes still queued on the OPFS backend. The /// `Session` is cloned (a snapshot copy); the container is shared. #[derive(Clone)] pub struct Gdd { pub(crate) session: Session, - pub(crate) working: Arc, + pub(crate) working: AnyContainer, pub(crate) layout: L, /// In-memory copy of the manifest, kept authoritative since `Gdd` is its sole writer. Holds the /// per-payload codecs so the persist path never probes the filesystem, keeping it fully read-free @@ -101,10 +100,10 @@ impl Gdd { /// Create a fresh, empty working copy at `path` bound to `peer`. Writes a default manifest /// and session state; the caller fills in editor metadata via [`Gdd::update_manifest`]. - pub async fn create(path: &Path, peer: PeerId, document_uuid: u64, editor_version: String, stdlib_version: String) -> Result { + pub fn create(path: &Path, peer: PeerId, document_uuid: u64, editor_version: String, stdlib_version: String) -> Result { let working = AnyContainer::Folder(FolderBackend::create(path)?); let layout = L::default(); - Self::create_in(working, layout, peer, document_uuid, editor_version, stdlib_version).await + Self::create_in(working, layout, peer, document_uuid, editor_version, stdlib_version) } } @@ -124,10 +123,14 @@ impl Gdd { /// Backend-agnostic open. Splits out so tests can supply a [`document_container::backends::memory::MemoryBackend`]. /// /// # Errors - /// [`Error::WrongFormat`] / [`Error::UnsupportedVersion`] if the manifest fails validation, plus - /// the usual [`Error::Read`] / [`Error::Codec`] / [`Error::Crdt`] if a payload is malformed. + /// [`Error::MissingManifest`] if the container holds no manifest, [`Error::WrongFormat`] / [`Error::UnsupportedVersion`] + /// if it fails validation, plus the usual [`Error::Read`] / [`Error::Codec`] / [`Error::Crdt`] if a payload is malformed. pub async fn open_in(working: AnyContainer, layout: L) -> Result { - let manifest: Manifest = io::read_single(&working, layout.manifest_basename(), MANIFEST_CODEC).await?; + let manifest: Manifest = match io::read_single(&working, layout.manifest_basename(), MANIFEST_CODEC).await { + Ok(manifest) => manifest, + Err(io::ReadError::NotFound { .. }) => return Err(Error::MissingManifest), + Err(error) => return Err(error.into()), + }; validate_manifest(&manifest)?; let codecs = manifest.codecs; @@ -163,7 +166,7 @@ impl Gdd { Ok(Self { session, - working: Arc::new(working), + working, layout, manifest, view_settings: session_state.view_settings, @@ -173,7 +176,7 @@ impl Gdd { /// Backend-agnostic create. Records the working-copy default codecs (see `DEFAULT_*_CODEC`) in /// the manifest and writes each payload with its recorded codec. - pub async fn create_in(working: AnyContainer, layout: L, peer: PeerId, document_uuid: u64, editor_version: String, stdlib_version: String) -> Result { + pub fn create_in(working: AnyContainer, layout: L, peer: PeerId, document_uuid: u64, editor_version: String, stdlib_version: String) -> Result { let manifest = Manifest::new(document_uuid, editor_version, stdlib_version); let codecs = manifest.codecs; io::write_single(&working, layout.manifest_basename(), MANIFEST_CODEC, &manifest)?; @@ -185,7 +188,7 @@ impl Gdd { Ok(Self { session, - working: Arc::new(working), + working, layout, manifest, view_settings: std::collections::BTreeMap::new(), @@ -285,11 +288,8 @@ impl Gdd { } /// Drop the session and return the working-copy container + layout. - /// Intended for test code that needs to reopen against the same container; panics if the container - /// is still shared by a `Gdd` clone (tests don't clone before calling this). pub fn into_storage(self) -> (AnyContainer, L) { - let working = Arc::try_unwrap(self.working).unwrap_or_else(|_| panic!("into_storage called while the working-copy container is still shared by a Gdd clone")); - (working, self.layout) + (self.working, self.layout) } /// Resolve every proto-node declaration referenced by the registry or its history into a @@ -320,16 +320,7 @@ impl Gdd { declarations } - /// Store the legacy `.graphite` document bytes verbatim inside the working copy (dual-write soak). - /// Synchronous (hot-path safe via `write_non_blocking`): called at the autosave boundary alongside - /// the registry snapshot. The bytes are opaque to `Gdd` — it never deserializes them. - // TODO: Add feature gate for legacy embedding - pub fn store_legacy_document(&self, bytes: &[u8]) -> Result<(), ContainerError> { - self.working.write_non_blocking(self.layout.legacy_path(), bytes) - } - - /// Read back the embedded legacy `.graphite` document, if present. The compare-on-open oracle and - /// the recovery fallback both go through here. `None` when no legacy blob was ever written. + /// Read the embedded legacy `.graphite` document, if present. pub async fn read_legacy_document(&self) -> Option { self.working.read(self.layout.legacy_path()).await.ok() } diff --git a/document/format/src/resource.rs b/document/format/src/resource.rs index c1b08da3955..b47241f6e0f 100644 --- a/document/format/src/resource.rs +++ b/document/format/src/resource.rs @@ -4,7 +4,6 @@ #[cfg(not(target_family = "wasm"))] use std::path::Path; -use std::sync::Arc; use document_container::{AnyContainer, AsyncContainer, ByteHolder, ContainerError}; use graphene_resource::ResourceFuture; @@ -39,7 +38,7 @@ impl Gdd { #[cfg(not(target_family = "wasm"))] pub fn add_resource_from_path(&mut self, id: document_graph_storage::ResourceId, hash: ResourceHash, src: &Path) -> Result<(), Error> { let dest_path = self.layout.resource_path(&hash); - if let AnyContainer::Folder(folder) = self.working.as_ref() { + if let AnyContainer::Folder(folder) = &self.working { let full = folder.root().join(&dest_path); if let Some(parent) = full.parent() { std::fs::create_dir_all(parent).map_err(ContainerError::Io)?; @@ -102,7 +101,7 @@ impl LoadResource for Gdd { } } -pub struct ResourceProxy(Arc, T); +pub struct ResourceProxy(AnyContainer, T); impl LoadResource for ResourceProxy { fn load(&self, hash: ResourceHash) -> ResourceFuture<'_> { diff --git a/document/format/tests/open_create.rs b/document/format/tests/open_create.rs index 4bdd43d7bd7..de5c684bc92 100644 --- a/document/format/tests/open_create.rs +++ b/document/format/tests/open_create.rs @@ -44,7 +44,7 @@ fn create_in_round_trips_empty_document() { futures::executor::block_on(async { let container = empty_container(); - let created = match GddV1::create_in(container, GddV1Layout, PeerId(7), 0xFEED, "editor-x".into(), "stdlib-x".into()).await { + let created = match GddV1::create_in(container, GddV1Layout, PeerId(7), 0xFEED, "editor-x".into(), "stdlib-x".into()) { Ok(gdd) => gdd, Err(error) => panic!("create_in failed: {error:?}"), }; @@ -82,9 +82,7 @@ fn open_in_rejects_wrong_format_magic() { #[test] fn manifest_returns_what_create_in_wrote() { futures::executor::block_on(async { - let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(13), 0xC0FFEE, "ed-1.2".into(), "std-0.7".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(13), 0xC0FFEE, "ed-1.2".into(), "std-0.7".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); assert_eq!(gdd.session().peer(), PeerId(13)); @@ -99,9 +97,7 @@ fn manifest_returns_what_create_in_wrote() { #[test] fn update_manifest_changes_visible_after_reopen() { futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); gdd.update_manifest(|m| m.editor_version = "ed-NEW".into()) .unwrap_or_else(|error| panic!("update_manifest failed: {error:?}")); @@ -116,9 +112,7 @@ fn update_manifest_changes_visible_after_reopen() { #[test] fn apply_hot_op_persists_to_hot_log_and_survives_reopen() { futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); // AddNetwork on the root network. Idempotent at apply, so two hot ops applied in sequence // produces one network in the registry. @@ -143,9 +137,7 @@ fn apply_hot_op_persists_to_hot_log_and_survives_reopen() { #[test] fn retire_moves_eligible_hot_ops_to_history_and_keeps_rest() { futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); // Two hot ops: one with low timestamp (will retire), one with high (will stay). let early = HotOp { @@ -189,9 +181,7 @@ fn retire_moves_eligible_hot_ops_to_history_and_keeps_rest() { #[test] fn last_broadcast_rev_persists_across_reopen() { futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); // Retire one op so there is a real retired rev to mark as published. let op = HotOp { @@ -220,9 +210,7 @@ fn export_folder_round_trips_through_open() { use document_format::{ExportFormat, ExportOptions}; futures::executor::block_on(async { - let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(3), 0xAB, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(3), 0xAB, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("export"); @@ -250,9 +238,7 @@ fn export_zip_round_trips_via_deserialize() { use document_format::{ExportFormat, ExportOptions}; futures::executor::block_on(async { - let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(4), 0xCD, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(4), 0xCD, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("doc.gdd.zip"); @@ -277,9 +263,7 @@ fn export_rejects_invalid_options() { use document_format::{ExportFormat, ExportOptions}; futures::executor::block_on(async { - let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xEF, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xEF, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("nope"); @@ -302,9 +286,7 @@ fn resource_round_trip_add_read_remove() { use graphene_resource::{ResourceHash, ResourceId}; futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(99), 0xCAFE, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(99), 0xCAFE, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let payload = b"deadbeef cafe babe"; let hash = ResourceHash::from(&payload[..]); @@ -334,9 +316,7 @@ fn resource_survives_reopen() { use graphene_resource::{ResourceHash, ResourceId}; futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(7), 0xC0DE, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(7), 0xC0DE, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let payload = b"persistent bytes"; let hash = ResourceHash::from(&payload[..]); @@ -365,9 +345,7 @@ fn resource_from_path_uses_fs_copy_on_folder_backend() { // Need a folder-backed working copy to exercise the fs::copy path. let working_dir = tempfile::tempdir().unwrap(); let working = AnyContainer::Folder(FolderBackend::create(working_dir.path()).unwrap()); - let mut gdd = GddV1::create_in(working, GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(working, GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); // Source file outside the working copy. let payload = b"external resource bytes"; @@ -391,9 +369,7 @@ fn export_carries_resources() { use graphene_resource::{ResourceHash, ResourceId}; futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(2), 0xBC, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(2), 0xBC, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let payload = b"exported resource"; let hash = ResourceHash::from(&payload[..]); @@ -421,9 +397,7 @@ fn embed_all_resources_materializes_link_only_resource() { use graphene_resource::{DataSource, ResourceHash, ResourceId, ResourceRegistry}; futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(8), 0xF00D, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(8), 0xF00D, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); // A resource whose only source is a URL, resolved to a hash. The bytes live solely in the // byte store; the working copy never holds them. @@ -488,9 +462,7 @@ fn export_materializes_embedded_resource_from_byte_store() { use graphene_resource::{DataSource, ResourceHash, ResourceId, ResourceRegistry}; futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(9), 0xBEEF, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(9), 0xBEEF, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); // An Embedded resource whose bytes live only in the byte store, not the working copy. let payload = b"embedded bytes in the cache"; @@ -531,9 +503,7 @@ fn export_round_trips_unretired_hot_ops() { use graphene_resource::{DataSource, ResourceId, ResourceRegistry}; futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(3), 0xF15E, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(3), 0xF15E, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let payload = b"hot resource bytes"; let hash = graphene_resource::ResourceHash::from(&payload[..]); @@ -586,9 +556,7 @@ fn open_in_rejects_future_format_version() { #[test] fn create_in_records_default_codecs_in_manifest() { futures::executor::block_on(async { - let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let codecs = gdd.manifest().codecs; assert_eq!(codecs.registry, Codec::MessagePack); @@ -609,9 +577,7 @@ fn first_commit_registers_peer_and_survives_reopen() { use graphene_resource::ResourceRegistry; futures::executor::block_on(async { - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(21), 0xAB, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(21), 0xAB, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let network = NodeNetwork { exports: vec![NodeInput::node(core_types::uuid::NodeId(0), 0)], @@ -645,9 +611,7 @@ fn persist_path_writes_at_manifest_declared_codec_paths() { futures::executor::block_on(async { use document_container::AsyncContainer; - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); let hot_op = HotOp { op: RegistryDelta::AddNetwork { @@ -698,9 +662,7 @@ fn declarations_round_trip_through_byte_store() { ..Default::default() }; - let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()) - .await - .unwrap_or_else(|error| panic!("create_in failed: {error:?}")); + let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into()).unwrap_or_else(|error| panic!("create_in failed: {error:?}")); // Commit: declaration bytes flow into the byte store, not the Gdd container. let byte_store = HashMapResourceStorage::new(); diff --git a/editor/src/application.rs b/editor/src/application.rs index ade13460987..cd2f2c3f418 100644 --- a/editor/src/application.rs +++ b/editor/src/application.rs @@ -14,14 +14,14 @@ impl Editor { environment: Environment, uuid_random_seed: u64, resource_storage: Arc, - working_copy_root: Option, + document_store: Arc, mut application_io: PlatformApplicationIo, wake: Wake, ) -> Self { ENVIRONMENT.set(environment).expect("Editor shoud only be initialized once"); graphene_std::uuid::set_uuid_seed(uuid_random_seed); - let mut dispatcher = Dispatcher::new(resource_storage, working_copy_root); + let mut dispatcher = Dispatcher::new(resource_storage, document_store); dispatcher.message_handlers.future_message_handler.set_wake(wake); application_io.inject_resource_proxy(dispatcher.message_handlers.resource_storage_message_handler.resources()); crate::node_graph_executor::replace_application_io(application_io); diff --git a/editor/src/dispatcher.rs b/editor/src/dispatcher.rs index 6ddd599179c..9aa8bc31889 100644 --- a/editor/src/dispatcher.rs +++ b/editor/src/dispatcher.rs @@ -89,16 +89,19 @@ const DEBUG_MESSAGE_BLOCK_LIST: &[MessageDiscriminant] = &[ const DEBUG_MESSAGE_ENDING_BLOCK_LIST: &[&str] = &["PointerMove", "PointerOutsideViewport", "Overlays", "Draw", "CurrentTime", "Time"]; impl Dispatcher { - pub fn new(resource_storage: Arc, working_copy_root: Option) -> Self { + pub fn new(resource_storage: Arc, document_store: Arc) -> Self { let mut s = Self::default(); s.message_handlers.resource_storage_message_handler = ResourceStorageMessageHandler::new(resource_storage); - s.message_handlers.portfolio_message_handler.set_working_copy_root(working_copy_root); + s.message_handlers.portfolio_message_handler.set_document_store(document_store); s } #[cfg(test)] pub fn with_executor(executor: crate::node_graph_executor::NodeGraphExecutor) -> Self { - let mut s = Self::new(Arc::new(graph_craft::application_io::resource::HashMapResourceStorage::new()), None); + let mut s = Self::new( + Arc::new(graph_craft::application_io::resource::HashMapResourceStorage::new()), + Arc::new(document_container::store::MemoryStore::default()), + ); s.message_handlers.portfolio_message_handler = PortfolioMessageHandler::with_executor(executor); s } diff --git a/editor/src/lib.rs b/editor/src/lib.rs index 5acc1c2ed16..56e89e7f4e7 100644 --- a/editor/src/lib.rs +++ b/editor/src/lib.rs @@ -19,3 +19,5 @@ pub mod node_graph_executor; pub mod test_utils; pub mod utility_traits; pub mod utility_types; + +pub use document_container::store as document_store; diff --git a/editor/src/messages/frontend/frontend_message.rs b/editor/src/messages/frontend/frontend_message.rs index f309f51d617..454deb336e6 100644 --- a/editor/src/messages/frontend/frontend_message.rs +++ b/editor/src/messages/frontend/frontend_message.rs @@ -118,22 +118,9 @@ pub enum FrontendMessage { filename: String, }, TriggerPersistenceReadState, - TriggerPersistenceReadDocument { - #[serde(rename = "documentId")] - document_id: DocumentId, - }, TriggerPersistenceWriteState { state: PersistedState, }, - TriggerPersistenceDeleteDocument { - #[serde(rename = "documentId")] - document_id: DocumentId, - }, - TriggerPersistenceWriteDocument { - #[serde(rename = "documentId")] - document_id: DocumentId, - document: String, - }, TriggerOpenLaunchDocuments, TriggerLoadPreferences, TriggerSavePreferences { diff --git a/editor/src/messages/frontend/utility_types.rs b/editor/src/messages/frontend/utility_types.rs index b0a8a7655fa..27e2791ac8b 100644 --- a/editor/src/messages/frontend/utility_types.rs +++ b/editor/src/messages/frontend/utility_types.rs @@ -1,7 +1,5 @@ use std::path::PathBuf; -use graph_craft::application_io::resource::ResourceHash; - use crate::messages::portfolio::document::utility_types::document_metadata::LayerNodeIdentifier; use crate::messages::portfolio::utility_types::WorkspacePanelLayout; use crate::messages::prelude::*; @@ -12,9 +10,6 @@ pub struct DocumentInfo { pub id: DocumentId, pub name: String, #[serde(default)] - #[cfg_attr(feature = "wasm", tsify(type = "unknown"))] - pub resources: Option>, - #[serde(default)] pub path: Option, pub is_saved: bool, } diff --git a/editor/src/messages/portfolio/document/document_history.rs b/editor/src/messages/portfolio/document/document_history.rs index c0d5535c97a..1e9bbdf263b 100644 --- a/editor/src/messages/portfolio/document/document_history.rs +++ b/editor/src/messages/portfolio/document/document_history.rs @@ -112,6 +112,12 @@ impl DocumentHistory { self.needs_whole_document_stage = true; } + /// Detach the `Gdd` working copy, dropping its declaration cache with it. + pub fn clear_storage(&mut self) { + self.storage = None; + self.declarations.clear(); + } + /// Retire the pending staged hot ops into durable Gdd history as one undo unit. Called at each undo-step /// boundary (a new `StartTransaction`) and before undo/redo, so the per-`CommitTransaction` staging /// coalesces into one interaction aligned with the legacy step. No-op while unmounted. @@ -132,7 +138,6 @@ impl DocumentHistory { interface: &NodeNetworkInterface, registry: &ResourceRegistry, view_settings: BTreeMap, - legacy_document: &str, byte_store: &dyn ResourceStorage, ) { let needs_whole_document_stage = self.needs_whole_document_stage; @@ -161,7 +166,7 @@ impl DocumentHistory { } } - self.persist_view_state(interface, view_settings, legacy_document); + self.persist_view_state(interface, view_settings); } /// Converts the whole document and stages the difference from what the working copy holds. @@ -198,9 +203,8 @@ impl DocumentHistory { Ok(constructed.declarations.decoded) } - /// The parts of a commit that are not the graph: the per-peer view settings, and the legacy bytes the - /// dual-write soak validates the new format against. - fn persist_view_state(&mut self, interface: &NodeNetworkInterface, view_settings: BTreeMap, legacy_document: &str) { + /// The parts of a commit that are not the graph: the per-peer and per-network view settings. + fn persist_view_state(&mut self, interface: &NodeNetworkInterface, view_settings: BTreeMap) { let Some(storage) = self.storage.as_mut() else { return }; let network_view_settings = storage @@ -217,10 +221,6 @@ impl DocumentHistory { { log::error!("Persisting per-network view settings failed: {error}"); } - - if let Err(error) = storage.store_legacy_document(legacy_document.as_bytes()) { - log::error!("Embedding legacy document into working copy failed: {error}"); - } } /// Move the `Gdd` undo/redo cursor along the retired interaction chain, flushing any open interaction diff --git a/editor/src/messages/portfolio/document/document_message_handler.rs b/editor/src/messages/portfolio/document/document_message_handler.rs index c6b0fdb6e18..ec1da9aaf1e 100644 --- a/editor/src/messages/portfolio/document/document_message_handler.rs +++ b/editor/src/messages/portfolio/document/document_message_handler.rs @@ -32,6 +32,7 @@ use crate::messages::tool::tool_messages::select_tool::SelectToolPointerKeys; use crate::messages::tool::tool_messages::tool_prelude::Key; use crate::messages::tool::utility_types::ToolType; use crate::node_graph_executor::NodeGraphExecutor; +use document_container::AnyContainer; use document_graph_storage::Declarations; use glam::{DAffine2, DVec2}; use graph_craft::application_io::resource::ResourceId; @@ -149,6 +150,10 @@ pub struct DocumentMessageHandler { /// Undo/redo state: the legacy snapshot stacks plus the `Gdd` working-copy cursor. #[serde(skip)] history: DocumentHistory, + /// The document's container in the document store. `None` for a document created this session until its store entry opens. + #[serde(skip)] + #[derivative(Debug = "ignore")] + pub(crate) container: Option, /// Hash of the document snapshot that was most recently saved to disk by the user. #[serde(skip)] saved_hash: Option, @@ -200,6 +205,7 @@ impl Default for DocumentMessageHandler { breadcrumb_network_path: Vec::new(), selection_network_path: Vec::new(), history: DocumentHistory::default(), + container: None, saved_hash: None, auto_saved_hash: None, layer_range_selection_reference: None, @@ -407,8 +413,8 @@ impl MessageHandler> for DocumentMes responses.add(NodeGraphMessage::SelectedNodesSet { nodes: vec![] }); self.layer_range_selection_reference = None; } - DocumentMessage::DocumentHistoryBackward => self.undo_with_history(viewport, preferences.validate_storage_round_trip, responses), - DocumentMessage::DocumentHistoryForward => self.redo_with_history(viewport, preferences.validate_storage_round_trip, responses), + DocumentMessage::DocumentHistoryBackward => self.undo_with_history(document_id, viewport, preferences.validate_storage_round_trip, responses), + DocumentMessage::DocumentHistoryForward => self.redo_with_history(document_id, viewport, preferences.validate_storage_round_trip, responses), DocumentMessage::DocumentStructureChanged => { if layers_panel_open { self.network_interface.load_structure(); @@ -2028,6 +2034,11 @@ impl DocumentMessageHandler { self.history.set_storage(storage, declarations); } + /// Detach the `Gdd` working copy. + pub fn clear_storage(&mut self) { + self.history.clear_storage(); + } + /// Retire the pending staged hot ops into durable Gdd history as one undo unit. pub(crate) fn retire_storage_interaction(&mut self) { self.history.retire_storage_interaction(); @@ -2050,20 +2061,19 @@ impl DocumentMessageHandler { return; } - let (view_settings, legacy_document) = self.storage_side_channels(); - self.history - .stage_snapshot(&deltas, &self.network_interface, &self.resources.registry, view_settings, legacy_document.as_str(), byte_store); + let view_settings = self.storage_view_settings(); + self.history.stage_snapshot(&deltas, &self.network_interface, &self.resources.registry, view_settings, byte_store); if validate { self.history.verify_round_trip(&self.network_interface, &self.resources.registry); } } - /// The per-peer view settings and legacy bytes that ride along with either kind of staging. - fn storage_side_channels(&self) -> (std::collections::BTreeMap, String) { + /// The per-peer view settings that ride along with either kind of staging. + fn storage_view_settings(&self) -> std::collections::BTreeMap { use crate::messages::portfolio::document::utility_types::network_interface::storage_metadata::DocumentSettings; - let view_settings = DocumentSettings { + DocumentSettings { document_ptz: &self.document_ptz, render_mode: &self.render_mode, overlays_visibility: &self.overlays_visibility_settings, @@ -2071,9 +2081,7 @@ impl DocumentMessageHandler { snapping_state: &self.snapping_state, collapsed: &self.collapsed, } - .to_view_map(); - - (view_settings, self.serialize_document()) + .to_view_map() } /// Restore `view_settings` map into the document. @@ -2104,11 +2112,15 @@ impl DocumentMessageHandler { } } - /// Move the `Gdd` undo/redo cursor and swap in the interface rebuilt from it. `had_oracle` records - /// whether the legacy snapshot already applied, so the rebuild can be compared against it. - fn drive_storage_undo_redo(&mut self, had_oracle: bool, undo: bool, validate: bool, responses: &mut VecDeque) { + /// Move the `Gdd` undo/redo cursor, swap in the interface rebuilt from it, and autosave so the legacy + /// document matches the cursor. `had_oracle` records whether the legacy snapshot already applied, so the + /// rebuild can be compared against it. + fn drive_storage_undo_redo(&mut self, document_id: DocumentId, had_oracle: bool, undo: bool, validate: bool, responses: &mut VecDeque) { match self.history.move_cursor(undo) { - Ok(rebuilt) => self.apply_gdd_cursor_rebuild(rebuilt, had_oracle, validate, responses), + Ok(rebuilt) => { + self.apply_gdd_cursor_rebuild(rebuilt, had_oracle, validate, responses); + responses.add(PortfolioMessage::AutoSaveDocument { document_id }); + } Err(CursorMoveError::NotMoved) => {} // Without the legacy snapshot the interface stayed put, so the cursor has to follow it back. Err(CursorMoveError::RebuildFailed) => { @@ -2431,7 +2443,7 @@ impl DocumentMessageHandler { paths } - pub fn undo_with_history(&mut self, viewport: &ViewportMessageHandler, validate: bool, responses: &mut VecDeque) { + pub fn undo_with_history(&mut self, document_id: DocumentId, viewport: &ViewportMessageHandler, validate: bool, responses: &mut VecDeque) { let legacy_applied = if let Some(previous_network) = self.undo(viewport, responses) { self.history.push_redo(previous_network); true @@ -2439,7 +2451,7 @@ impl DocumentMessageHandler { false }; - self.drive_storage_undo_redo(legacy_applied, true, validate, responses); + self.drive_storage_undo_redo(document_id, legacy_applied, true, validate, responses); } /// Installs a history snapshot as the active network interface, carrying over the current view state and structure load, and returns the replaced interface. @@ -2473,7 +2485,7 @@ impl DocumentMessageHandler { Some(previous_network) } - pub fn redo_with_history(&mut self, viewport: &ViewportMessageHandler, validate: bool, responses: &mut VecDeque) { + pub fn redo_with_history(&mut self, document_id: DocumentId, viewport: &ViewportMessageHandler, validate: bool, responses: &mut VecDeque) { let legacy_applied = if let Some(previous_network) = self.redo(viewport, responses) { self.history.push_undo(previous_network); true @@ -2481,7 +2493,7 @@ impl DocumentMessageHandler { false }; - self.drive_storage_undo_redo(legacy_applied, false, validate, responses); + self.drive_storage_undo_redo(document_id, legacy_applied, false, validate, responses); } pub fn redo(&mut self, viewport: &ViewportMessageHandler, responses: &mut VecDeque) -> Option { diff --git a/editor/src/messages/portfolio/document/storage_tests/lifecycle_tests.rs b/editor/src/messages/portfolio/document/storage_tests/lifecycle_tests.rs new file mode 100644 index 00000000000..fa6e4ec8176 --- /dev/null +++ b/editor/src/messages/portfolio/document/storage_tests/lifecycle_tests.rs @@ -0,0 +1,215 @@ +//! Document store lifecycle: discovery, session restore, legacy/storage reconciliation, mode changes, and removal. + +use crate::messages::frontend::utility_types::PersistedState; +use crate::messages::portfolio::document_storage_io::document_key; +use crate::messages::prelude::*; +use crate::test_utils::EditorTestUtils; +use document_container::{AnyContainer, AsyncContainer}; + +fn dispatch(editor: &mut EditorTestUtils, message: impl Into) -> Vec { + editor.editor.handle_message(message) +} + +fn portfolio(editor: &EditorTestUtils) -> &PortfolioMessageHandler { + &editor.editor.dispatcher.message_handlers.portfolio_message_handler +} + +fn restart(editor: &mut EditorTestUtils, state: PersistedState) { + dispatch(editor, PortfolioMessage::DestroyAllDocuments); + dispatch(editor, PersistentStateMessage::LoadState { state }); +} + +fn new_document(editor: &mut EditorTestUtils, name: &str) -> DocumentId { + dispatch(editor, PortfolioMessage::NewDocumentWithName { name: name.into() }); + portfolio(editor).active_document_id.unwrap() +} + +async fn container(editor: &EditorTestUtils, document_id: DocumentId) -> AnyContainer { + portfolio(editor).document_store().open(document_key(document_id)).await.unwrap() +} + +async fn stored_ids(editor: &EditorTestUtils) -> Vec { + portfolio(editor).document_store().list().await.unwrap().into_iter().map(|key| DocumentId(key.0)).collect() +} + +#[tokio::test] +async fn stored_documents_are_discovered_without_session_state() { + let mut editor = EditorTestUtils::create(); + let one = new_document(&mut editor, "One"); + let two = new_document(&mut editor, "Two"); + + restart(&mut editor, PersistedState::default()); + + let state = portfolio(&editor).persisted_state_snapshot(); + let mut ids: Vec<_> = state.documents.iter().map(|info| info.id).collect(); + ids.sort(); + let mut expected = vec![one, two]; + expected.sort(); + assert_eq!(ids, expected); + assert_ne!( + state.documents[0].name, state.documents[1].name, + "names are not stored in the legacy file, so discovered documents get distinct generated ones" + ); + assert_eq!(portfolio(&editor).documents.len(), 2); + assert!(portfolio(&editor).active_document().is_some()); +} + +#[tokio::test] +async fn session_state_restores_order_names_and_selection() { + let mut editor = EditorTestUtils::create(); + new_document(&mut editor, "First"); + let selected = new_document(&mut editor, "Second"); + let state = portfolio(&editor).persisted_state_snapshot(); + let expected: Vec<_> = state.documents.iter().map(|info| (info.id, info.name.clone())).collect(); + + restart(&mut editor, state); + + let restored = portfolio(&editor).persisted_state_snapshot(); + assert_eq!(restored.documents.iter().map(|info| (info.id, info.name.clone())).collect::>(), expected); + assert_eq!(portfolio(&editor).active_document_id, Some(selected)); +} + +#[tokio::test] +async fn session_entries_without_stored_content_are_dropped() { + let mut editor = EditorTestUtils::create(); + let kept = new_document(&mut editor, "Kept"); + let mut state = portfolio(&editor).persisted_state_snapshot(); + let mut ghost = state.documents[0].clone(); + ghost.id = DocumentId(0xdead); + state.documents.push(ghost); + + restart(&mut editor, state); + + let ids: Vec<_> = portfolio(&editor).persisted_state_snapshot().documents.iter().map(|info| info.id).collect(); + assert_eq!(ids, vec![kept]); +} + +#[tokio::test] +async fn legacy_only_container_gets_storage() { + let mut editor = EditorTestUtils::create(); + let id = DocumentId(0xabc); + let container = container(&editor, id).await; + container.write("legacy.graphite", DocumentMessageHandler::default().serialize_document().as_bytes()).await.unwrap(); + + restart(&mut editor, PersistedState::default()); + + assert!(portfolio(&editor).documents.contains_key(&id)); + assert!(editor.active_document().storage().is_some()); + assert!(container.exists("manifest.json").await); + assert!(container.exists("legacy.graphite").await); +} + +#[tokio::test] +async fn legacy_mode_reads_legacy_only_and_leaves_storage_files_alone() { + let mut editor = EditorTestUtils::create(); + dispatch(&mut editor, PreferencesMessage::SaveAsGdd { enabled: false }); + let id = new_document(&mut editor, "Legacy"); + let container = container(&editor, id).await; + assert!(!container.exists("manifest.json").await); + container.write("manifest.json", b"broken").await.unwrap(); + + restart(&mut editor, PersistedState::default()); + + assert!(portfolio(&editor).documents.contains_key(&id)); + assert!(editor.active_document().storage().is_none()); + assert_eq!(container.read("manifest.json").await.unwrap().as_slice(), b"broken"); +} + +#[tokio::test] +async fn storage_only_container_recovers_and_writes_legacy() { + let mut editor = EditorTestUtils::create(); + let id = new_document(&mut editor, "Storage"); + let container = container(&editor, id).await; + container.remove("legacy.graphite").await.unwrap(); + + restart(&mut editor, PersistedState::default()); + + assert!(portfolio(&editor).documents.contains_key(&id)); + assert!(editor.active_document().storage().is_some()); + assert!(container.exists("legacy.graphite").await); +} + +#[tokio::test] +async fn legacy_wins_over_stale_storage_and_history_is_kept() { + let mut editor = EditorTestUtils::create(); + let id = new_document(&mut editor, "Changed"); + let container = container(&editor, id).await; + let before = editor.active_document().serialize_document(); + let network_before = editor.active_document().document_network().clone(); + editor.draw_rect(0., 0., 100., 100.).await; + dispatch(&mut editor, PortfolioMessage::AutoSaveDocument { document_id: id }); + container.write("legacy.graphite", before.as_bytes()).await.unwrap(); + + restart(&mut editor, PersistedState::default()); + + assert_eq!(editor.active_document().document_network(), &network_before); + let storage = editor.active_document().storage().expect("storage attached"); + assert!(storage.can_undo(), "the stale stored state stays reachable as history"); +} + +#[tokio::test] +async fn unchanged_content_keeps_cursor_and_redo_across_restart() { + let mut editor = EditorTestUtils::create(); + let id = new_document(&mut editor, "Undo"); + let network_before = editor.active_document().document_network().clone(); + editor.draw_rect(0., 0., 100., 100.).await; + dispatch(&mut editor, PortfolioMessage::AutoSaveDocument { document_id: id }); + + editor.handle_message(DocumentMessage::Undo).await; + assert_eq!(editor.active_document().document_network(), &network_before); + assert!(editor.active_document().storage().unwrap().can_redo()); + assert!(editor.active_document().is_auto_saved(), "the rebuild completion autosaves the cursor's network"); + + restart(&mut editor, PersistedState::default()); + + assert_eq!(editor.active_document().document_network(), &network_before); + let storage = editor.active_document().storage().expect("storage attached"); + assert!(storage.can_redo(), "reopening identical content must not stage a new interaction"); + assert!(!storage.can_undo()); +} + +#[tokio::test] +async fn closing_removes_the_stored_document() { + let mut editor = EditorTestUtils::create(); + let id = new_document(&mut editor, "Closed"); + assert_eq!(stored_ids(&editor).await, vec![id]); + + dispatch(&mut editor, PortfolioMessage::CloseDocument { document_id: id }); + + assert!(stored_ids(&editor).await.is_empty()); + restart(&mut editor, PersistedState::default()); + assert!(portfolio(&editor).persisted_state_snapshot().documents.is_empty()); +} + +#[tokio::test] +async fn storage_preference_attaches_and_drops_sessions() { + let mut editor = EditorTestUtils::create(); + dispatch(&mut editor, PreferencesMessage::SaveAsGdd { enabled: false }); + let id = new_document(&mut editor, "Toggle"); + let container = container(&editor, id).await; + assert!(editor.active_document().storage().is_none()); + assert!(container.exists("legacy.graphite").await); + + dispatch(&mut editor, PreferencesMessage::SaveAsGdd { enabled: true }); + assert!(editor.active_document().storage().is_some()); + assert!(container.exists("manifest.json").await); + + dispatch(&mut editor, PreferencesMessage::SaveAsGdd { enabled: false }); + assert!(editor.active_document().storage().is_none()); +} + +#[tokio::test] +async fn unreadable_document_is_reported_and_kept_until_discarded() { + let mut editor = EditorTestUtils::create(); + let id = DocumentId(7); + container(&editor, id).await.write("legacy.graphite", b"not a document").await.unwrap(); + + restart(&mut editor, PersistedState::default()); + + assert!(!portfolio(&editor).documents.contains_key(&id)); + assert!(portfolio(&editor).persisted_state_snapshot().documents.is_empty()); + assert_eq!(stored_ids(&editor).await, vec![id]); + + dispatch(&mut editor, FailedDocumentsMessage::DiscardFailedToLoadDocuments); + assert!(stored_ids(&editor).await.is_empty()); +} diff --git a/editor/src/messages/portfolio/document/storage_tests/mod.rs b/editor/src/messages/portfolio/document/storage_tests/mod.rs index ed7024f6352..98ccd1ec1f5 100644 --- a/editor/src/messages/portfolio/document/storage_tests/mod.rs +++ b/editor/src/messages/portfolio/document/storage_tests/mod.rs @@ -4,6 +4,7 @@ //! //! Shared fixtures live in [`test_support`]. +mod lifecycle_tests; mod metadata_tests; mod round_trip_tests; mod test_support; diff --git a/editor/src/messages/portfolio/document/storage_tests/round_trip_tests.rs b/editor/src/messages/portfolio/document/storage_tests/round_trip_tests.rs index 5c6e85a597b..2cf3660df94 100644 --- a/editor/src/messages/portfolio/document/storage_tests/round_trip_tests.rs +++ b/editor/src/messages/portfolio/document/storage_tests/round_trip_tests.rs @@ -158,9 +158,7 @@ async fn edit_after_open_commits_cleanly() { // reopened registry: the editor's .gdd-open path. let byte_store = HashMapResourceStorage::new(); let source = editor.active_document(); - let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()) - .await - .expect("create_in"); + let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()).expect("create_in"); let source_network = source.network_interface.document_network().clone(); let source_view = StorageMetadataView::new(&source.network_interface); gdd.commit_from_runtime(&source_network, &source_view, &source.resources.registry, &byte_store) @@ -210,9 +208,7 @@ async fn edit_after_open_commits_cleanly() { #[tokio::test] async fn gdd_undo_redo_walks_interactions() { let byte_store = HashMapResourceStorage::new(); - let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()) - .await - .expect("create_in"); + let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()).expect("create_in"); // Commit the active document's current runtime state as one interaction. async fn commit_interaction(gdd: &mut GddV1, document: &DocumentMessageHandler, byte_store: &HashMapResourceStorage) { @@ -263,9 +259,7 @@ async fn gdd_undo_redo_walks_interactions() { #[tokio::test] async fn reopen_after_undo_restores_consistent_registry() { let byte_store = HashMapResourceStorage::new(); - let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()) - .await - .expect("create_in"); + let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()).expect("create_in"); async fn commit_interaction(gdd: &mut GddV1, document: &DocumentMessageHandler, byte_store: &HashMapResourceStorage) { let network = document.network_interface.document_network().clone(); @@ -334,7 +328,7 @@ async fn live_undo_shadows_storage_cursor() { editor.new_document().await; editor.draw_rect(0., 0., 100., 100.).await; - let byte_store = mount_in_memory_storage(&mut editor).await; + let byte_store = mount_in_memory_storage(&mut editor); // Capture the loaded state as the base interaction, then make a real edit (fires CommitTransaction). editor.active_document_mut().commit_storage_snapshot(&byte_store, true); let before_edit = editor.active_document().network_interface.document_network().clone(); @@ -387,9 +381,7 @@ async fn gdd_archive_round_trips_view_settings() { use document_graph_storage::attr::session::doc; let byte_store = HashMapResourceStorage::new(); - let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()) - .await - .expect("create_in"); + let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()).expect("create_in"); // Stage a distinctive PTZ into the working copy's `view_settings`, as `commit_storage_snapshot` does. let mut ptz = crate::messages::portfolio::document::utility_types::misc::PTZ::default(); @@ -437,9 +429,7 @@ async fn per_network_navigation_round_trips_via_session_not_registry() { let expected_pan = editor.active_document().network_interface.node_graph_ptz(&[]).unwrap().pan; // Commit the document into a fresh `Gdd`, collecting the per-network view state the editor persists. - let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()) - .await - .expect("create_in"); + let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()).expect("create_in"); let document = editor.active_document(); let network = document.network_interface.document_network().clone(); let view = StorageMetadataView::new(&document.network_interface); @@ -472,14 +462,14 @@ async fn per_network_navigation_round_trips_via_session_not_registry() { } /// Mirror the real "new document, draw a rect, press undo" flow: mount storage and capture the -/// new-document base as the mount-time snapshot (as `DocumentStorageMounted` does), then draw a rect +/// new-document base as the attach-time snapshot (as `StorageAttached` does), then draw a rect /// (one `CommitTransaction` interaction) and undo. The shadow must reproduce the legacy-restored interface. #[tokio::test] async fn live_undo_new_document_draw_rect() { let mut editor = EditorTestUtils::create(); editor.new_document().await; - let byte_store = mount_in_memory_storage(&mut editor).await; + let byte_store = mount_in_memory_storage(&mut editor); // Mount-time snapshot: capture the new-document graph as the base interaction. editor.active_document_mut().commit_storage_snapshot(&byte_store, true); @@ -569,7 +559,7 @@ async fn undo_image_paste_resources_subset_of_runtime() { let mut editor = EditorTestUtils::create(); editor.new_document().await; - let byte_store = mount_in_memory_storage(&mut editor).await; + let byte_store = mount_in_memory_storage(&mut editor); editor.active_document_mut().commit_storage_snapshot(&byte_store, true); editor.handle_message(paste_named_image()).await; @@ -601,7 +591,7 @@ async fn undo_twice_steps_cursor_two_interactions() { let mut editor = EditorTestUtils::create(); editor.new_document().await; - let byte_store = mount_in_memory_storage(&mut editor).await; + let byte_store = mount_in_memory_storage(&mut editor); editor.active_document_mut().commit_storage_snapshot(&byte_store, true); let base = editor.active_document().network_interface.document_network().clone(); @@ -674,10 +664,8 @@ fn assert_cursor_matches_runtime(document: &DocumentMessageHandler, at: &str) { /// Mount a fresh in-memory `Gdd` onto the active document so `commit_storage_snapshot` (the real /// autosave path) runs against it. Returns the byte store the document's resources resolve through. -async fn mount_in_memory_storage(editor: &mut EditorTestUtils) -> HashMapResourceStorage { - let gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0x5EED, "test".into(), "test".into()) - .await - .expect("create_in"); +fn mount_in_memory_storage(editor: &mut EditorTestUtils) -> HashMapResourceStorage { + let gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0x5EED, "test".into(), "test".into()).expect("create_in"); editor.active_document_mut().set_storage(gdd, Default::default()); HashMapResourceStorage::new() } @@ -701,7 +689,7 @@ async fn demo_artwork_edit_autosaves_and_round_trips() { }) .await; - let byte_store = mount_in_memory_storage(&mut editor).await; + let byte_store = mount_in_memory_storage(&mut editor); // First autosave: captures the loaded document. `verify_storage_round_trip` panics on drift. editor.active_document_mut().commit_storage_snapshot(&byte_store, true); @@ -769,7 +757,7 @@ async fn none_fill_survives_document_reopen() { // Reopen through the editor's real open path, which runs the document migrations let serialized = editor.active_document().serialize_document(); editor - .handle_message(PortfolioMessage::OpenDocumentFile { + .handle_message(PortfolioMessage::OpenLegacyDocumentFile { document_name: None, document_path: None, document_serialized_content: serialized, @@ -792,7 +780,7 @@ async fn legacy_four_input_fill_migrates_to_the_split_transform_shape() { let mut editor = EditorTestUtils::create(); editor - .handle_message(PortfolioMessage::OpenDocumentFile { + .handle_message(PortfolioMessage::OpenLegacyDocumentFile { document_name: None, document_path: None, document_serialized_content: LEGACY_DOCUMENT.to_string(), @@ -850,7 +838,7 @@ async fn eight_input_fill_migrates_the_spread_input_into_the_ramp() { let mut editor = EditorTestUtils::create(); editor - .handle_message(PortfolioMessage::OpenDocumentFile { + .handle_message(PortfolioMessage::OpenLegacyDocumentFile { document_name: None, document_path: None, document_serialized_content: EIGHT_INPUT_DOCUMENT.to_string(), diff --git a/editor/src/messages/portfolio/document/storage_tests/test_support.rs b/editor/src/messages/portfolio/document/storage_tests/test_support.rs index bc511de7041..f58a4e93d84 100644 --- a/editor/src/messages/portfolio/document/storage_tests/test_support.rs +++ b/editor/src/messages/portfolio/document/storage_tests/test_support.rs @@ -56,9 +56,7 @@ pub struct RoundTrip { pub async fn round_trip_through_gdd(document: &DocumentMessageHandler) -> RoundTrip { let byte_store = HashMapResourceStorage::new(); - let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()) - .await - .expect("create_in"); + let mut gdd = GddV1::create_in(AnyContainer::Memory(MemoryBackend::new()), GddV1Layout, PeerId(1), 0xABCD, "test".into(), "test".into()).expect("create_in"); let network = document.network_interface.document_network().clone(); let view = StorageMetadataView::new(&document.network_interface); diff --git a/editor/src/messages/portfolio/document_storage_io.rs b/editor/src/messages/portfolio/document_storage_io.rs index d68d65c8837..8ac356ad776 100644 --- a/editor/src/messages/portfolio/document_storage_io.rs +++ b/editor/src/messages/portfolio/document_storage_io.rs @@ -1,215 +1,359 @@ -//! Asynchronous `.gdd` working-copy IO, spawned by `PortfolioMessageHandler` as `FutureMessage`s: -//! building/opening containers and opening `.gdd` archives into documents. -//! The `validate` flag is the `validate_storage_round_trip` preference; when -//! set, the registry build is compared against the legacy oracle (logged, not fatal) for the soak. - -use document_container::AnyContainer; -use document_format::{Error as DocumentFormatError, GddV1, GddV1Layout}; -use document_graph_storage::Declarations; -use graph_craft::application_io::resource::ResourceStorage; -use graph_craft::document::NodeNetwork; +//! Document store I/O spawned by `PortfolioMessageHandler` as `FutureMessage`s. A future writes only into a +//! container no registered document owns yet; the message loop is the sole writer once a document is +//! registered, and a completion for a document closed meanwhile removes the store entry. use super::document::DocumentMessageHandler; -use super::document::diff_networks; +use super::document::utility_types::document_metadata::LayerNodeIdentifier; +use super::document::utility_types::error::EditorError; +use super::document::utility_types::network_interface::OutputConnector; use super::document::utility_types::network_interface::storage_metadata::{apply_network_view_settings, build_interface_from_storage, network_ids_from_entries}; -use super::document_migration::document_migration_string_preprocessing; +use super::document_migration::*; use super::portfolio_message::PortfolioMessage; +use crate::application::{GRAPHITE_GIT_COMMIT_HASH, generate_uuid}; use crate::messages::message::Message; use crate::messages::portfolio::document::utility_types::misc::DocumentId; use crate::messages::resource_storage::ResourcesHandle; +use document_container::store::{DocumentKey, DocumentStore, MemoryStore}; +use document_container::{AnyContainer, AsyncContainer}; +use document_format::{Error as DocumentFormatError, GddV1, GddV1Layout, Layout, ReadError}; +use document_graph_storage::{Declarations, PeerId}; +use graph_craft::application_io::resource::{DataSource, ResourceHash, ResourceStorage}; +use std::path::PathBuf; +use std::sync::Arc; + +#[derive(Clone)] +pub struct DocumentStoreHandle(Arc); + +impl DocumentStoreHandle { + pub fn new(store: Arc) -> Self { + Self(store) + } +} + +impl std::ops::Deref for DocumentStoreHandle { + type Target = dyn DocumentStore; + + fn deref(&self) -> &(dyn DocumentStore + 'static) { + self.0.as_ref() + } +} + +impl Default for DocumentStoreHandle { + fn default() -> Self { + Self(Arc::new(MemoryStore::default())) + } +} + +impl std::fmt::Debug for DocumentStoreHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DocumentStoreHandle").finish_non_exhaustive() + } +} + +pub(crate) fn document_key(document_id: DocumentId) -> DocumentKey { + DocumentKey(document_id.0) +} -/// Build the container backend at `path` (folder on native, OPFS directory name on web) or in-memory -/// when `None`. Also returns whether an existing working copy was found (its `manifest.json` is present). -async fn build_per_document_container(path: Option<&std::path::Path>) -> Result<(AnyContainer, bool), DocumentFormatError> { - let Some(path) = path else { - return Ok((AnyContainer::Memory(document_container::backends::memory::MemoryBackend::new()), false)); +pub(crate) fn legacy_path() -> &'static str { + GddV1Layout.legacy_path() +} + +pub(super) async fn list_stored_documents(store: DocumentStoreHandle) -> Message { + let document_ids = match store.list().await { + Ok(keys) => Some(keys.into_iter().map(|key| DocumentId(key.0)).collect()), + Err(error) => { + log::error!("Listing the document store failed: {error}"); + None + } }; + PortfolioMessage::StoredDocumentsListed { document_ids }.into() +} - #[cfg(not(target_family = "wasm"))] - { - use document_container::backends::folder::FolderBackend; - let exists = path.join("manifest.json").is_file(); - let backend = if exists { FolderBackend::open(path)? } else { FolderBackend::create(path)? }; - Ok((AnyContainer::Folder(backend), exists)) - } - - #[cfg(target_family = "wasm")] - { - use document_container::AsyncContainer; - use document_container::backends::opfs::OpfsBackend; - let directory_name = path - .to_str() - .ok_or(DocumentFormatError::Container(document_container::ContainerError::InvalidPath(path.display().to_string())))?; - let backend = OpfsBackend::open(directory_name).await?; - let exists = backend.exists("manifest.json").await; - Ok((AnyContainer::Opfs(backend), exists)) - } -} - -/// Open the existing working copy at `path`, or create a fresh one bound to `peer` (in-memory when -/// `path` is `None`). Returns the working copy plus whether it was reopened (vs freshly created); only a -/// reopen has independently-stored state worth comparing against the legacy load. -pub(super) async fn build_or_open_working_copy( - path: Option<&std::path::Path>, - peer: document_graph_storage::PeerId, - document_uuid: u64, - version: String, -) -> Result<(GddV1, bool), DocumentFormatError> { - let (container, exists) = build_per_document_container(path).await?; - - let gdd = if exists { - GddV1::open_in(container, GddV1Layout).await? +pub(super) async fn load_stored_document(store: DocumentStoreHandle, document_id: DocumentId, resources: ResourcesHandle, reset_nodes: bool, legacy_only: bool) -> Message { + let (document, gdd) = match store.open(document_key(document_id)).await { + Ok(container) => load_from_container(container, document_id, &resources, reset_nodes, legacy_only).await, + Err(error) => { + log::error!("Opening the stored document {document_id:?} failed: {error}"); + (None, None) + } + }; + let declarations = load_declarations(gdd.as_ref(), &resources).await; + PortfolioMessage::StoredDocumentLoaded { + document_id, + document: document.map(Box::new), + gdd: gdd.map(Box::new), + declarations, + } + .into() +} + +/// Opens the container for a document created in this session. +pub(super) async fn attach_container(store: DocumentStoreHandle, document_id: DocumentId, resources: ResourcesHandle, legacy_only: bool) -> Message { + let container = match store.open(document_key(document_id)).await { + Ok(container) => container, + Err(error) => { + log::error!("Opening storage for {document_id:?} failed: {error}"); + return PortfolioMessage::StorageAttached { + document_id, + container: None, + gdd: None, + declarations: Declarations::new(), + } + .into(); + } + }; + let gdd = if !legacy_only { + open_storage_or_fallback_to_legacy(container.clone(), document_id).await } else { - GddV1::create_in(container, GddV1Layout, peer, document_uuid, version.clone(), version).await? + None }; - Ok((gdd, exists)) + let declarations = load_declarations(gdd.as_ref(), &resources).await; + PortfolioMessage::StorageAttached { + document_id, + container: Some(container), + gdd: gdd.map(Box::new), + declarations, + } + .into() } -/// `FutureMessage` that opens a `.gdd` archive into a document, delivered via -/// [`PortfolioMessage::GddDocumentLoaded`]. See [`build_document_from_gdd`] for the build itself. -pub(super) async fn open_gdd_document( - working_copy_root: Option, +/// Opens the storage of a document that already has its container. +pub(super) async fn open_storage(container: AnyContainer, document_id: DocumentId, resources: ResourcesHandle) -> Message { + let gdd = open_storage_or_fallback_to_legacy(container.clone(), document_id).await; + let declarations = load_declarations(gdd.as_ref(), &resources).await; + PortfolioMessage::StorageAttached { + document_id, + container: Some(container), + gdd: gdd.map(Box::new), + declarations, + } + .into() +} + +pub(super) async fn remove_stored_document(store: DocumentStoreHandle, document_id: DocumentId) -> Message { + if let Err(error) = store.remove(document_key(document_id)).await { + log::error!("Removing the stored document {document_id:?} failed: {error}"); + } + Message::NoOp +} + +/// Materializes a `.gdd` archive into a fresh store container and loads it like a stored document. +pub(super) async fn open_document_file( + store: DocumentStoreHandle, document_id: DocumentId, document_name: Option, - document_path: Option, + document_path: Option, content: Vec, - store_handle: ResourcesHandle, - validate: bool, + resources: ResourcesHandle, + reset_nodes: bool, + legacy_only: bool, ) -> Message { - let path = working_copy_root.map(|root| root.join(format!("{:x}", document_id.0))); - - let document = build_document_from_gdd(path.as_deref(), &content, &store_handle, document_id, validate).await; - Message::Portfolio(PortfolioMessage::GddDocumentLoaded { + let materialized = async { + let mut container = store.open(document_key(document_id)).await.map_err(|error| error.to_string())?; + document_container::archive::open_auto(&content, &mut container).map_err(|error| error.to_string())?; + Ok::<_, String>(container) + } + .await; + let (document, gdd) = match materialized { + Ok(container) => load_from_container(container, document_id, &resources, reset_nodes, legacy_only).await, + Err(error) => { + log::error!("Opening the document archive for {document_id:?} failed: {error}"); + (None, None) + } + }; + let declarations = load_declarations(gdd.as_ref(), &resources).await; + PortfolioMessage::DocumentFileLoaded { document_id, document_name, document_path, document: document.map(Box::new), - }) + gdd: gdd.map(Box::new), + declarations, + } + .into() } -/// Core of the `.gdd` open: archive -> working copy -> `Gdd` -> runtime interface. The registry build is -/// authoritative; the embedded legacy blob is the soak oracle and the fallback if the build fails. -/// Returns `None` only if neither the build nor the legacy fallback worked. -async fn build_document_from_gdd(path: Option<&std::path::Path>, content: &[u8], store_handle: &impl ResourceStorage, document_id: DocumentId, validate: bool) -> Option { - let (container, _exists) = match build_per_document_container(path).await { - Ok(result) => result, - Err(error) => { - log::error!("Opening .gdd for {document_id:?}: failed to build working copy container: {error}"); - return None; - } +/// Builds the document from its legacy file, or from its storage when that file is missing or unreadable. +async fn load_from_container(container: AnyContainer, document_id: DocumentId, resources: &ResourcesHandle, reset_nodes: bool, legacy_only: bool) -> (Option, Option) { + let legacy = match container.read(legacy_path()).await { + Ok(bytes) => String::from_utf8(bytes.as_slice().to_vec()) + .map_err(|error| error.to_string()) + .and_then(|text| load_legacy_document(text, resources, reset_nodes).map_err(|error| error.to_string())), + Err(error) => Err(error.to_string()), }; + match legacy { + Ok(mut document) => { + document.set_auto_save_state(true); + document.container = Some(container.clone()); + let gdd = if !legacy_only { open_storage_or_fallback_to_legacy(container, document_id).await } else { None }; + (Some(document), gdd) + } + Err(legacy_error) => { + let gdd = match GddV1::open_in(container.clone(), GddV1Layout).await { + Ok(gdd) => gdd, + Err(storage_error) => { + log::error!("Loading {document_id:?} failed. Legacy: {legacy_error}. Storage: {storage_error}"); + return (None, None); + } + }; + log::warn!("Recovering {document_id:?} from storage because its legacy file could not load: {legacy_error}"); + let Some(mut document) = build_document_from_storage(gdd, resources, document_id).await else { + return (None, None); + }; + document.container = Some(container); + if legacy_only { + document.clear_storage(); + } + (Some(document), None) + } + } +} - let gdd = match GddV1::open_from_archive(content, container, GddV1Layout).await { - Ok(gdd) => gdd, - Err(error) => { - log::error!("Opening .gdd for {document_id:?}: failed to open archive: {error}"); +async fn open_storage_or_fallback_to_legacy(container: AnyContainer, document_id: DocumentId) -> Option { + match GddV1::open_in(container.clone(), GddV1Layout).await { + Ok(gdd) => return Some(gdd), + Err(DocumentFormatError::MissingManifest) => {} + // Container I/O and newer-format errors keep the existing files; anything else is broken data the legacy document replaces. + Err(error @ (DocumentFormatError::Container(_) | DocumentFormatError::Read(ReadError::Container(_)) | DocumentFormatError::UnsupportedVersion { .. })) => { + log::error!("Opening storage for {document_id:?} failed: {error}"); return None; } - }; + Err(error) => log::warn!("Storage for {document_id:?} is unusable and will be rebuilt from the legacy document: {error}"), + } + let version = GRAPHITE_GIT_COMMIT_HASH.to_string(); + let created = async { + for path in container.list("").await? { + if path != legacy_path() { + container.remove(&path).await?; + } + } + GddV1::create_in(container, GddV1Layout, PeerId(generate_uuid()), document_id.0, version.clone(), version) + } + .await; + match created { + Ok(gdd) => Some(gdd), + Err(error) => { + log::error!("Creating storage for {document_id:?} failed: {error}"); + None + } + } +} + +/// Decoded proto-node declarations for every node `gdd` or its history references, from the global cache. +async fn load_declarations(gdd: Option<&GddV1>, resources: &ResourcesHandle) -> Declarations { + match gdd { + Some(gdd) => gdd.declarations(resources).await, + None => Declarations::new(), + } +} - // Extract archived resource bytes into the global cache so declarations + runtime resolve. +/// Storage session -> runtime interface. Resources stored in the container are hydrated into the global cache first. +async fn build_document_from_storage(gdd: GddV1, byte_store: &ResourcesHandle, document_id: DocumentId) -> Option { match gdd.resource_hashes().await { Ok(hashes) => { for hash in hashes { match gdd.read_resource(&hash).await { Ok(holder) => { - store_handle.store(holder.as_slice()); + byte_store.store(holder.as_slice()); } - Err(error) => log::error!("Opening .gdd for {document_id:?}: failed to read resource {hash}: {error}"), - } - } - } - Err(error) => log::error!("Opening .gdd for {document_id:?}: failed to list resources: {error}"), - } - - let legacy_document = gdd - .read_legacy_document() - .await - .and_then(|holder| String::from_utf8(holder.as_slice().to_vec()).ok()) - .and_then(|serialized| DocumentMessageHandler::deserialize_document(&document_migration_string_preprocessing(serialized)).ok()); - - let declarations = gdd.declarations(store_handle).await; - let interface = match gdd.registry().to_runtime_with_full_metadata(&declarations) { - Ok((network, node_entries, network_entries)) => { - let network_ids = network_ids_from_entries(&network_entries); - match build_interface_from_storage(network, node_entries, network_entries) { - Ok(mut interface) => { - // Per-network view state lives in `session.json`, not the registry, so restore it here. - apply_network_view_settings(&mut interface, &network_ids, gdd.network_view_settings()); - Some(interface) - } - Err(error) => { - log::error!("Opening .gdd for {document_id:?}: failed to build interface: {error}"); - None + Err(error) => log::error!("Opening document {document_id:?}: failed to read resource {hash}: {error}"), } } } + Err(error) => log::error!("Opening document {document_id:?}: failed to list resources: {error}"), + } + + let declarations = gdd.declarations(byte_store).await; + let (network, node_entries, network_entries) = match gdd.registry().to_runtime_with_full_metadata(&declarations) { + Ok(result) => result, Err(error) => { - log::error!("Opening .gdd for {document_id:?}: failed to convert registry to runtime: {error}"); - None + log::error!("Opening document {document_id:?}: failed to convert registry to runtime: {error}"); + return None; } }; - - // Soak compare against the legacy oracle, normalizing the benign generic-vs-resolved `call_argument`. - if let Some(interface) = interface { - if validate && let Some(legacy) = &legacy_document { - let mut built = interface.document_network().clone(); - let mut oracle = legacy.network_interface.document_network().clone(); - normalize_call_arguments_for_compare(&mut built); - normalize_call_arguments_for_compare(&mut oracle); - if built != oracle { - log::error!( - "Open-.gdd divergence for {document_id:?}: registry-built document differs from the embedded legacy blob\n{}", - diff_networks(&oracle, &built) - ); - } + let network_ids = network_ids_from_entries(&network_entries); + let mut interface = match build_interface_from_storage(network, node_entries, network_entries) { + Ok(interface) => interface, + Err(error) => { + log::error!("Opening document {document_id:?}: failed to build interface: {error}"); + return None; } - return Some(DocumentMessageHandler::from_storage(interface, gdd, declarations, String::new(), None)); - } + }; + apply_network_view_settings(&mut interface, &network_ids, gdd.network_view_settings()); - log::warn!("Opening .gdd for {document_id:?}: registry build failed, falling back to embedded legacy document"); - if legacy_document.is_none() { - log::error!("Opening .gdd for {document_id:?}: no embedded legacy document to fall back to"); - } - legacy_document + let mut document = DocumentMessageHandler::from_storage(interface, gdd, declarations, String::new(), None); + document.finalize_storage_load(); + Some(document) } -/// Soak check that the reopened `.gdd`'s stored registry, converted back to a runtime network, matches -/// the legacy load. Logs divergence only (legacy stays authoritative); runs once per open. -pub(super) fn compare_storage_against_runtime(gdd: &GddV1, legacy_network: &NodeNetwork, declarations: &Declarations, document_id: DocumentId) { - let mut candidate = match gdd.registry().to_runtime_with_metadata(declarations) { - Ok((network, _entries)) => network, - Err(error) => { - log::error!("Compare-on-open for {document_id:?}: .gdd registry failed to convert to runtime: {error}"); +/// Deserializes a legacy `.graphite` document, running the migrations and importing its embedded resources. +pub(super) fn load_legacy_document(document_serialized_content: String, byte_store: &impl ResourceStorage, reset_node_definitions_on_open: bool) -> Result { + // Upgrade the document being opened to use fresh copies of all nodes + let reset_node_definitions_on_open = reset_node_definitions_on_open || document_migration_reset_node_definition(&document_serialized_content); + // Upgrade the document being opened with string replacements on the original JSON + let document_serialized_content = document_migration_string_preprocessing(document_serialized_content); + // Upgrade resources from being referend by hash to beeing referened by ID + let (document_serialized_content, resource_hash_to_id_migration_map) = document_migration_replace_resources_referenced_by_hash(document_serialized_content); + + let mut document = DocumentMessageHandler::deserialize_document(&document_serialized_content)?; + // Upgrade the document's nodes to be compatible with the latest version + document_migration_upgrades(&mut document, reset_node_definitions_on_open); + + // Load the document's embedded resources into the resource storage + std::mem::take(&mut document.resources.embedded).into_iter().for_each(|(hash, resource)| { + let data: Arc<[u8]> = Arc::from(resource.as_ref()); + if ResourceHash::from(data.as_ref()) != hash { + log::error!("Resource hash mismatch for resource with hash {hash}"); return; } - }; + byte_store.store(&data); - // Normalize the benign generic-vs-resolved `call_argument` on both sides before comparing. - let mut legacy = legacy_network.clone(); - normalize_call_arguments_for_compare(&mut legacy); - normalize_call_arguments_for_compare(&mut candidate); + // TODO: Eventually remove this document upgrade code + // Register any resources that were previously referenced by hash + if let Some(id) = resource_hash_to_id_migration_map.get(&hash) + && !document.resources.registry.contains(id) + { + document.resources.registry.resolve(id, hash); + document.resources.registry.push_source_back(id, DataSource::Embedded); + } + }); - if candidate != legacy { - log::error!( - "Compare-on-open divergence for {document_id:?}: the reopened .gdd does not match the legacy load\n{}", - diff_networks(&legacy, &candidate) - ); + // Ensure each node has the metadata for its inputs + for (node_id, node, path) in document.network_interface.document_network().clone().recursive_nodes() { + document.network_interface.validate_input_metadata(node_id, node, &path); + document.network_interface.validate_output_names(node_id, node, &path); } -} -/// Recursively rewrite a concrete `Context` `call_argument` back to the generic `T` storage authored, so -/// the generic-vs-resolved difference (recomputed during compilation) compares equal. Other types untouched. -fn normalize_call_arguments_for_compare(network: &mut NodeNetwork) { - use graph_craft::Type; + // Ensure layers are positioned as stacks if they are upstream siblings of another layer + document.network_interface.load_structure(); + let all_layers = LayerNodeIdentifier::ROOT_PARENT.descendants(document.network_interface.document_metadata()).collect::>(); + for layer in all_layers { + let Some((downstream_node, input_index)) = document + .network_interface + .outward_wires(&[]) + .and_then(|outward_wires| outward_wires.get(&OutputConnector::primary_output(layer.to_node()))) + .and_then(|outward_wires| outward_wires.first()) + .and_then(|input_connector| input_connector.node_id().map(|node_id| (node_id, input_connector.input_index()))) + else { + continue; + }; - let context = graph_craft::concrete!(graphene_std::Context); - for node in network.nodes.values_mut() { - if node.call_argument == context { - node.call_argument = Type::Generic(std::borrow::Cow::Borrowed("T")); - } - if let graph_craft::document::DocumentNodeImplementation::Network(inner) = &mut node.implementation { - normalize_call_arguments_for_compare(inner); + // If the downstream node is a layer and the input is the first input and the current layer is not in a stack + if input_index == 0 && document.network_interface.is_layer(&downstream_node, &[]) && !document.network_interface.is_stack(&layer.to_node(), &[]) { + // Ensure the layer is horizontally aligned with the downstream layer to prevent changing the layout of old files + let (Some(layer_position), Some(downstream_position)) = (document.network_interface.position(&layer.to_node(), &[]), document.network_interface.position(&downstream_node, &[])) else { + log::error!("Could not get position for layer {:?} or downstream node {} when opening file", layer.to_node(), downstream_node); + continue; + }; + + if layer_position.x == downstream_position.x { + document.network_interface.set_stack_position_calculated_offset(&layer.to_node(), &downstream_node, &[]); + } } } + + document.network_interface.discard_deltas(); + document.require_whole_document_stage(); + + Ok(document) } diff --git a/editor/src/messages/portfolio/failed_documents/failed_documents_message_handler.rs b/editor/src/messages/portfolio/failed_documents/failed_documents_message_handler.rs index 0cf3acaf5de..0285ae3654f 100644 --- a/editor/src/messages/portfolio/failed_documents/failed_documents_message_handler.rs +++ b/editor/src/messages/portfolio/failed_documents/failed_documents_message_handler.rs @@ -2,29 +2,26 @@ use crate::consts::FILE_EXTENSION; use crate::messages::dialog::simple_dialogs; use crate::messages::frontend::utility_types::{DocumentInfo, FileFilter}; use crate::messages::layout::utility_types::layout_widget::DialogLayoutHolder; -use crate::messages::portfolio::persistent_state::PersistentStateMessage; +use crate::messages::portfolio::document_storage_io::{DocumentStoreHandle, document_key, legacy_path, remove_stored_document}; use crate::messages::prelude::*; +use document_container::AsyncContainer; #[derive(ExtractField)] pub struct FailedDocumentsMessageContext<'a> { - pub document_ids: &'a mut VecDeque, + pub document_store: &'a DocumentStoreHandle, } #[derive(Debug, Default, ExtractField)] pub struct FailedDocumentsMessageHandler { - /// Pairs of `(info, raw serialized content)` for autosaved documents that failed to deserialize. - /// The info entries are folded back into `persisted_state_snapshot` so their on-disk autosave files survive garbage collection. + /// Stored documents that failed to load. Their store entries stay until explicitly discarded. // TODO: Eventually remove this document upgrade code - failed_to_load_documents: HashMap, - /// In-flight count of autosaved-document loads from the initial startup batch; the batched failure dialog fires when this hits 0. - // TODO: Eventually remove this document upgrade code - pending_initial_autosave_loads: usize, + failed_to_load_documents: HashMap, } #[message_handler_data] impl MessageHandler> for FailedDocumentsMessageHandler { fn process_message(&mut self, message: FailedDocumentsMessage, responses: &mut VecDeque, context: FailedDocumentsMessageContext) { - let FailedDocumentsMessageContext { document_ids } = context; + let FailedDocumentsMessageContext { document_store } = context; match message { // TODO: Eventually remove this document upgrade code @@ -32,19 +29,15 @@ impl MessageHandler> f if self.failed_to_load_documents.is_empty() { return; } - let failed_document_names = self.failed_to_load_documents.values().map(|(info, _)| display_name_with_fallback(info)).collect(); + let failed_document_names = self.failed_to_load_documents.values().map(display_name_with_fallback).collect(); let dialog = simple_dialogs::FailedToLoadDocumentsDialog { failed_document_names }; dialog.send_dialog_to_frontend(responses); } // TODO: Eventually remove this document upgrade code FailedDocumentsMessage::DiscardFailedToLoadDocuments => { - let failed = std::mem::take(&mut self.failed_to_load_documents); - for document_id in failed.keys() { - document_ids.retain(|id| id != document_id); - responses.add(PersistentStateMessage::DeleteDocument { document_id: *document_id }); + for document_id in std::mem::take(&mut self.failed_to_load_documents).into_keys() { + responses.add(remove_stored_document(document_store.clone(), document_id)); } - responses.add(PortfolioMessage::UpdateOpenDocumentsList); - responses.add(PersistentStateMessage::WriteState); } // TODO: Eventually remove this document upgrade code FailedDocumentsMessage::DownloadFailedToLoadDocuments => { @@ -52,60 +45,22 @@ impl MessageHandler> f return; } - let mut used_names = HashSet::new(); - let files: Vec<(String, Vec)> = self - .failed_to_load_documents - .values() - .map(|(info, content)| { - let stem = sanitize_filename_stem(&info.name).unwrap_or_else(|| format!("document-{:x}", info.id.0)); - - // A name that was already given out, even as another document's numbered copy, takes the next free number - let mut unique = format!("{stem}.{FILE_EXTENSION}"); - let mut copy_number = 1; - while !used_names.insert(unique.clone()) { - unique = format!("{stem} ({copy_number}).{FILE_EXTENSION}"); - copy_number += 1; - } - - (unique, content.as_bytes().to_vec()) - }) - .collect(); - - const FOLDER_NAME: &str = "Graphite Recovered Documents"; - - if files.len() == 1 { - let (filename, content) = files.into_iter().next().expect("just checked there's one entry"); - responses.add(FrontendMessage::TriggerSaveFile { - name: filename, - folder: None, - filters: vec![FileFilter { - name: "Graphite Document".into(), - extensions: vec![FILE_EXTENSION.into()], - mime_types: Vec::new(), - }], - content: serde_bytes::ByteBuf::from(content), - }); - } else { - match build_recovery_zip(&files) { - Ok(zip_bytes) => responses.add(FrontendMessage::TriggerSaveFile { - name: format!("{FOLDER_NAME}.zip"), - folder: None, - filters: vec![FileFilter { - name: "Zip Archive".into(), - extensions: vec!["zip".into()], - mime_types: Vec::new(), - }], - content: serde_bytes::ByteBuf::from(zip_bytes), - }), - Err(e) => { - log::error!("Failed to build recovery zip: {e}"); - responses.add(DialogMessage::DisplayDialogError { - title: "Failed to download".to_string(), - description: format!("Could not bundle the failed documents for download.\n\n{e}"), - }); + let files = recovery_file_names(self.failed_to_load_documents.values()); + let store = document_store.clone(); + responses.add(async move { + let mut entries = Vec::new(); + for (document_id, filename) in files { + let content = match store.open(document_key(document_id)).await { + Ok(container) => container.read(legacy_path()).await.map(|bytes| bytes.as_slice().to_vec()), + Err(error) => Err(error), + }; + match content { + Ok(bytes) => entries.push((filename, bytes)), + Err(error) => log::error!("Reading the stored document {document_id:?} for recovery failed: {error}"), } } - } + recovery_download(entries) + }); } } } @@ -115,48 +70,87 @@ impl MessageHandler> f impl FailedDocumentsMessageHandler { // TODO: Eventually remove this document upgrade code - /// Keeps a document that failed to load, with its raw serialized content, so it can be offered for download. - pub(crate) fn record_failure(&mut self, document_id: DocumentId, info: DocumentInfo, serialized_content: String) { - self.failed_to_load_documents.insert(document_id, (info, serialized_content)); + pub(crate) fn record_failure(&mut self, document_id: DocumentId, info: DocumentInfo) { + self.failed_to_load_documents.insert(document_id, info); } +} - // TODO: Eventually remove this document upgrade code - /// Counts more autosaved documents of the startup batch whose loads the batched failure dialog waits for. - pub(crate) fn expect_autosave_loads(&mut self, count: usize) { - self.pending_initial_autosave_loads = self.pending_initial_autosave_loads.saturating_add(count); +// TODO: Eventually remove this document upgrade code +fn display_name_with_fallback(info: &DocumentInfo) -> String { + if info.name.trim().is_empty() { + format!("Untitled Document ({:x})", info.id.0) + } else { + info.name.clone() } +} - // TODO: Eventually remove this document upgrade code - /// Whether documents failed to load and none of the startup batch is still loading, so the failure dialog is due. - pub(crate) fn has_failures_to_report(&self) -> bool { - self.pending_initial_autosave_loads == 0 && !self.failed_to_load_documents.is_empty() - } +// TODO: Eventually remove this document upgrade code +fn recovery_file_names<'a>(infos: impl Iterator) -> Vec<(DocumentId, String)> { + let mut used_names = HashSet::new(); + infos + .map(|info| { + let stem = sanitize_filename_stem(&info.name).unwrap_or_else(|| format!("document-{:x}", info.id.0)); + + // A name that was already given out, even as another document's numbered copy, takes the next free number + let mut unique = format!("{stem}.{FILE_EXTENSION}"); + let mut copy_number = 1; + while !used_names.insert(unique.clone()) { + unique = format!("{stem} ({copy_number}).{FILE_EXTENSION}"); + copy_number += 1; + } - // TODO: Eventually remove this document upgrade code - /// The info of each document that failed to load. - pub(crate) fn failed_document_infos(&self) -> impl Iterator { - self.failed_to_load_documents.values().map(|(info, _)| info) + (info.id, unique) + }) + .collect() +} + +// TODO: Eventually remove this document upgrade code +fn recovery_download(files: Vec<(String, Vec)>) -> Message { + const FOLDER_NAME: &str = "Graphite Recovered Documents"; + + if files.is_empty() { + return DialogMessage::DisplayDialogError { + title: "Failed to download".to_string(), + description: "None of the failed documents could be read from storage.".to_string(), + } + .into(); } - // TODO: Eventually remove this document upgrade code - pub(crate) fn tick_autosave_load_progress(&mut self, responses: &mut VecDeque, failed: bool) { - if self.pending_initial_autosave_loads > 0 { - self.pending_initial_autosave_loads -= 1; - if self.has_failures_to_report() { - responses.add(FailedDocumentsMessage::ShowFailedToLoadDocumentsDialog); - } - } else if failed { - responses.add(FailedDocumentsMessage::ShowFailedToLoadDocumentsDialog); + if files.len() == 1 { + let (filename, content) = files.into_iter().next().expect("just checked there's one entry"); + return FrontendMessage::TriggerSaveFile { + name: filename, + folder: None, + filters: vec![FileFilter { + name: "Graphite Document".into(), + extensions: vec![FILE_EXTENSION.into()], + mime_types: Vec::new(), + }], + content: serde_bytes::ByteBuf::from(content), } + .into(); } -} -// TODO: Eventually remove this document upgrade code -fn display_name_with_fallback(info: &DocumentInfo) -> String { - if info.name.trim().is_empty() { - format!("Untitled Document ({:x})", info.id.0) - } else { - info.name.clone() + match build_recovery_zip(&files) { + Ok(zip_bytes) => FrontendMessage::TriggerSaveFile { + name: format!("{FOLDER_NAME}.zip"), + folder: None, + filters: vec![FileFilter { + name: "Zip Archive".into(), + extensions: vec!["zip".into()], + mime_types: Vec::new(), + }], + content: serde_bytes::ByteBuf::from(zip_bytes), + } + .into(), + Err(e) => { + log::error!("Failed to build recovery zip: {e}"); + DialogMessage::DisplayDialogError { + title: "Failed to download".to_string(), + description: format!("Could not bundle the failed documents for download.\n\n{e}"), + } + .into() + } } } @@ -229,25 +223,15 @@ mod tests { use super::*; #[test] - fn recovered_documents_sharing_a_name_with_a_numbered_copy_still_download() { - let mut handler = FailedDocumentsMessageHandler::default(); - for (id, name) in [(1, "Art"), (2, "Art"), (3, "Art (1)")] { - let info = DocumentInfo { - id: DocumentId(id), - name: name.to_string(), - resources: None, - path: None, - is_saved: true, - }; - handler.record_failure(DocumentId(id), info, "content".to_string()); - } - - let mut document_ids = VecDeque::new(); - let mut responses = VecDeque::new(); - let context = FailedDocumentsMessageContext { document_ids: &mut document_ids }; - handler.process_message(FailedDocumentsMessage::DownloadFailedToLoadDocuments, &mut responses, context); - - // A file name given out twice makes the archive fail to build, which shows an error dialog in place of the download - assert!(matches!(responses.front(), Some(Message::Frontend(FrontendMessage::TriggerSaveFile { .. })))); + fn recovered_documents_sharing_a_name_with_a_numbered_copy_get_distinct_file_names() { + let infos = [(1, "Art"), (2, "Art"), (3, "Art (1)")].map(|(id, name)| DocumentInfo { + id: DocumentId(id), + name: name.to_string(), + path: None, + is_saved: true, + }); + + let names: HashSet<_> = recovery_file_names(infos.iter()).into_iter().map(|(_, name)| name).collect(); + assert_eq!(names.len(), 3); } } diff --git a/editor/src/messages/portfolio/ingest/ingest_message_handler.rs b/editor/src/messages/portfolio/ingest/ingest_message_handler.rs index 3fa5ce8178c..77cbff7994b 100644 --- a/editor/src/messages/portfolio/ingest/ingest_message_handler.rs +++ b/editor/src/messages/portfolio/ingest/ingest_message_handler.rs @@ -97,7 +97,7 @@ impl MessageHandler for IngestMessageHandle let Ok(document_serialized_content) = String::from_utf8(data) else { return unsupported(responses); }; - responses.add(PortfolioMessage::OpenDocumentFile { + responses.add(PortfolioMessage::OpenLegacyDocumentFile { document_name: name, document_path, document_serialized_content, @@ -105,7 +105,7 @@ impl MessageHandler for IngestMessageHandle return; } DataType::Gdd => { - responses.add(PortfolioMessage::OpenGddDocument { + responses.add(PortfolioMessage::OpenDocumentFile { document_name: name, document_path, content: data, diff --git a/editor/src/messages/portfolio/persistent_state/persistent_state_message.rs b/editor/src/messages/portfolio/persistent_state/persistent_state_message.rs index 824afd1cafa..873b8666165 100644 --- a/editor/src/messages/portfolio/persistent_state/persistent_state_message.rs +++ b/editor/src/messages/portfolio/persistent_state/persistent_state_message.rs @@ -6,25 +6,5 @@ use crate::messages::prelude::*; pub enum PersistentStateMessage { ReadState, WriteState, - LoadState { - state: PersistedState, - }, - ReadDocument { - #[serde(rename = "documentId")] - document_id: DocumentId, - }, - WriteDocument { - #[serde(rename = "documentId")] - document_id: DocumentId, - document: String, - }, - DeleteDocument { - #[serde(rename = "documentId")] - document_id: DocumentId, - }, - LoadDocument { - #[serde(rename = "documentId")] - document_id: DocumentId, - document: String, - }, + LoadState { state: PersistedState }, } diff --git a/editor/src/messages/portfolio/persistent_state/persistent_state_message_handler.rs b/editor/src/messages/portfolio/persistent_state/persistent_state_message_handler.rs index 7e862662da4..b017eddc193 100644 --- a/editor/src/messages/portfolio/persistent_state/persistent_state_message_handler.rs +++ b/editor/src/messages/portfolio/persistent_state/persistent_state_message_handler.rs @@ -32,21 +32,6 @@ impl MessageHandler for P self.loaded = true; responses.add(PortfolioMessage::LoadPersistedState { state }); } - PersistentStateMessage::ReadDocument { document_id } => { - responses.add(FrontendMessage::TriggerPersistenceReadDocument { document_id }); - } - PersistentStateMessage::WriteDocument { document_id, document } => { - responses.add(FrontendMessage::TriggerPersistenceWriteDocument { document_id, document }); - } - PersistentStateMessage::DeleteDocument { document_id } => { - responses.add(FrontendMessage::TriggerPersistenceDeleteDocument { document_id }); - } - PersistentStateMessage::LoadDocument { document_id, document } => { - responses.add(PortfolioMessage::LoadDocumentContent { - document_id, - document_serialized_content: document, - }); - } } } diff --git a/editor/src/messages/portfolio/portfolio_message.rs b/editor/src/messages/portfolio/portfolio_message.rs index f453b3d321f..0f7cb3915cf 100644 --- a/editor/src/messages/portfolio/portfolio_message.rs +++ b/editor/src/messages/portfolio/portfolio_message.rs @@ -2,6 +2,7 @@ use super::document::utility_types::document_metadata::LayerNodeIdentifier; use super::persistent_state::PersistentStateMessage; use crate::messages::frontend::utility_types::{ExportBounds, FileType, PersistedState}; use crate::messages::prelude::*; +use document_container::AnyContainer; use std::path::PathBuf; #[impl_message(Message, Portfolio)] @@ -45,70 +46,76 @@ pub enum PortfolioMessage { DeleteDocument { document_id: DocumentId, }, - /// Delivers an asynchronously-built `Gdd` working copy, with the proto-node declarations its history - /// references, into its document. Emitted by the mount future spawned in `load_document`. The `mounted` - /// payload is non-serializable and a clone carries none (`clone_to_none`), so it travels exactly once. - /// `reopened` is true when an existing working copy was opened: the persisted cursor is trusted as-is - /// and the mount-time re-commit is skipped, since re-committing would stack a spurious interaction on - /// the restored cursor and make the first undo a no-op. - DocumentStorageMounted { - document_id: DocumentId, - reopened: bool, - #[serde(skip, default)] - #[derivative(Debug = "ignore", PartialEq = "ignore", Clone(clone_with = "clone_to_none"))] - mounted: Option>, - }, DestroyAllDocuments, EditorPreferences, GarbageCollectResources, + ResolveResources, ResolveDocumentResources { document_id: DocumentId, }, - ResolveResources, LoadPersistedState { state: PersistedState, }, - LoadDocumentContent { + StoredDocumentsListed { + document_ids: Option>, + }, + StoredDocumentLoaded { document_id: DocumentId, - document_serialized_content: String, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore", Clone(clone_with = "clone_to_none"))] + document: Option>, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore", Clone(clone_with = "clone_to_none"))] + gdd: Option>, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore")] + declarations: document_graph_storage::Declarations, }, NewDocumentWithName { name: String, }, - NextDocument, - OpenDocumentFile { + OpenLegacyDocumentFile { document_name: Option, document_path: Option, document_serialized_content: String, }, - /// Open a `.gdd` document container (archive bytes), building the runtime from its stored registry. - OpenGddDocument { + OpenDocumentFile { document_name: Option, document_path: Option, content: Vec, }, - /// Delivers a document built asynchronously from a `.gdd` archive (registry → runtime, working copy - /// mounted) into the portfolio. Travels once like [`DocumentStorageMounted`](Self::DocumentStorageMounted). - GddDocumentLoaded { + DocumentFileLoaded { document_id: DocumentId, document_name: Option, document_path: Option, #[serde(skip, default)] #[derivative(Debug = "ignore", PartialEq = "ignore", Clone(clone_with = "clone_to_none"))] document: Option>, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore", Clone(clone_with = "clone_to_none"))] + gdd: Option>, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore")] + declarations: document_graph_storage::Declarations, }, - LoadDocument { + StorageUpdated, + StorageAttached { document_id: DocumentId, - document_name: Option, - document_path: Option, - document_is_auto_saved: bool, - document_is_saved: bool, - document_serialized_content: String, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore", Clone(clone_with = "clone_to_none"))] + container: Option, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore", Clone(clone_with = "clone_to_none"))] + gdd: Option>, + #[serde(skip, default)] + #[derivative(Debug = "ignore", PartialEq = "ignore")] + declarations: document_graph_storage::Declarations, }, + NextDocument, + PrevDocument, CenterLayers { layers: Vec, }, - PrevDocument, ReorderDocument { document_id: DocumentId, new_index: usize, @@ -148,8 +155,7 @@ pub enum PortfolioMessage { UpdateOpenDocumentsList, } -/// Clone helper for non-serializable payloads: a cloned message carries none of the `Gdd` working copy, -/// its declarations, or the built document. +/// Clone helper for non-serializable payloads: a cloned completion message carries none. fn clone_to_none(_: &Option) -> Option { None } diff --git a/editor/src/messages/portfolio/portfolio_message_handler.rs b/editor/src/messages/portfolio/portfolio_message_handler.rs index 1887b41b383..a9f29964d56 100644 --- a/editor/src/messages/portfolio/portfolio_message_handler.rs +++ b/editor/src/messages/portfolio/portfolio_message_handler.rs @@ -2,7 +2,7 @@ use super::document::utility_types::document_metadata::LayerNodeIdentifier; use super::persistent_state::{PersistentStateMessage, PersistentStateMessageContext, PersistentStateMessageHandler}; use super::utility_types::PanelType; use crate::application::{Editor, generate_uuid}; -use crate::consts::{DEFAULT_DOCUMENT_NAME, FILE_EXTENSION, GDD_FILE_EXTENSION}; +use crate::consts::DEFAULT_DOCUMENT_NAME; use crate::messages::animation::TimingInformation; use crate::messages::dialog::simple_dialogs; use crate::messages::frontend::utility_types::{DocumentInfo, PersistedState}; @@ -12,19 +12,22 @@ use crate::messages::layout::utility_types::widget_prelude::*; use crate::messages::portfolio::document::DocumentMessageContext; use crate::messages::portfolio::document::graph_operation::utility_types::TransformIn; use crate::messages::portfolio::document::node_graph::document_node_definitions; -use crate::messages::portfolio::document::utility_types::network_interface::OutputConnector; -use crate::messages::portfolio::document_migration::*; -use crate::messages::portfolio::document_storage_io::{build_or_open_working_copy, compare_storage_against_runtime, open_gdd_document}; +use crate::messages::portfolio::document_storage_io::{ + DocumentStoreHandle, attach_container, legacy_path, list_stored_documents, load_legacy_document, load_stored_document, open_document_file, open_storage, remove_stored_document, +}; use crate::messages::preferences::SelectionMode; use crate::messages::prelude::*; +use crate::messages::resource_storage::ResourcesHandle; use crate::messages::tool::utility_types::{HintData, ToolType}; use crate::messages::viewport::ToPhysical; use crate::node_graph_executor::{ExportConfig, NodeGraphExecutor}; +use document_container::AsyncContainer; +use document_format::GddV1; +use document_graph_storage::Declarations; use glam::{DAffine2, DVec2}; -use graph_craft::application_io::resource::{DataSource, ResourceHash}; use graph_craft::document::NodeId; use graphene_std::renderer::Quad; -use std::path::PathBuf; +use std::path::Path; use std::sync::Arc; use std::vec; @@ -43,10 +46,7 @@ pub struct PortfolioMessageContext<'a> { #[derive(Debug, Default, ExtractField)] pub struct PortfolioMessageHandler { pub documents: HashMap, - unloaded_documents: HashMap, - /// Background eager loads whose trailing `SelectDocument` should be suppressed to keep focus on the user's active doc. - // TODO: Eventually remove this document upgrade code - pending_eager_loads: HashSet, + loading_documents: HashMap, document_ids: VecDeque, pub(crate) active_document_id: Option, failed_documents: FailedDocumentsMessageHandler, @@ -57,9 +57,10 @@ pub struct PortfolioMessageHandler { pub selection_mode: SelectionMode, pub reset_node_definitions_on_open: bool, pub workspace: WorkspaceMessageHandler, - working_copy_root: Option, - /// Number of documents not fully loaded. While nonzero, resource GC is skipped. + document_store: DocumentStoreHandle, pending_opens: usize, + restore_select: Option, + store_unavailable: bool, } #[message_handler_data] @@ -105,7 +106,7 @@ impl MessageHandler> for Portfolio self.persistent_state.process_message(message, responses, context); } PortfolioMessage::FailedDocuments(message) => { - let context = FailedDocumentsMessageContext { document_ids: &mut self.document_ids }; + let context = FailedDocumentsMessageContext { document_store: &self.document_store }; self.failed_documents.process_message(message, responses, context); } PortfolioMessage::Fonts(message) => { @@ -191,18 +192,16 @@ impl MessageHandler> for Portfolio } PortfolioMessage::AutoSaveActiveDocument => { if let Some(document_id) = self.active_document_id - && let Some(document) = self.active_document_mut() + && self.documents.contains_key(&document_id) { - document.set_auto_save_state(true); responses.add(PortfolioMessage::AutoSaveDocument { document_id }); } } PortfolioMessage::AutoSaveAllDocuments => { for document_id in self.document_ids.iter() { - if let Some(document) = self.documents.get_mut(document_id) + if let Some(document) = self.documents.get(document_id) && !document.is_auto_saved() { - document.set_auto_save_state(true); responses.add(PortfolioMessage::AutoSaveDocument { document_id: *document_id }); } } @@ -210,15 +209,23 @@ impl MessageHandler> for Portfolio responses.add(PortfolioMessage::GarbageCollectResources); } PortfolioMessage::AutoSaveDocument { document_id } => { - let validate = preferences.validate_storage_round_trip; let Some(document) = self.documents.get_mut(&document_id) else { return }; + let Some(container) = document.container.clone() else { return }; - document.commit_storage_snapshot(&resource_storage.resources_mut(), validate); - - responses.add(PersistentStateMessage::WriteDocument { - document_id, - document: document.serialize_document(), - }); + match container.write_non_blocking(legacy_path(), document.serialize_document().as_bytes()) { + Ok(()) => { + document.commit_storage_snapshot(&resource_storage.resources_mut(), preferences.validate_storage_round_trip); + document.set_auto_save_state(true); + } + Err(error) => { + document.set_auto_save_state(false); + log::error!("Autosave of {document_id:?} failed: {error}"); + responses.add(DialogMessage::DisplayDialogError { + title: "Autosave is not working".to_string(), + description: format!("Save your documents manually to avoid losing changes.\n\n{error}"), + }); + } + } responses.add(PersistentStateMessage::WriteState); } PortfolioMessage::CloseActiveDocumentWithConfirmation => { @@ -238,8 +245,8 @@ impl MessageHandler> for Portfolio HintData::clear_layout(responses); } - for document_id in &self.document_ids { - responses.add(PersistentStateMessage::DeleteDocument { document_id: *document_id }); + for document_id in self.document_ids.iter().copied().collect::>() { + responses.add(remove_stored_document(self.document_store.clone(), document_id)); } responses.add(PortfolioMessage::DestroyAllDocuments); @@ -264,7 +271,7 @@ impl MessageHandler> for Portfolio // Actually delete the document (delay to delete document is required to let the document and properties panel messages above get processed) responses.add(PortfolioMessage::DeleteDocument { document_id }); - responses.add(PersistentStateMessage::DeleteDocument { document_id }); + responses.add(remove_stored_document(self.document_store.clone(), document_id)); // Send the new list of document tab names responses.add(PortfolioMessage::UpdateOpenDocumentsList); @@ -292,8 +299,11 @@ impl MessageHandler> for Portfolio PortfolioMessage::DeleteDocument { document_id } => { let document_index = self.document_index(document_id); self.documents.remove(&document_id); - self.unloaded_documents.remove(&document_id); + self.loading_documents.remove(&document_id); self.document_ids.remove(document_index); + if self.restore_select == Some(document_id) { + self.restore_select = None; + } if self.document_ids.is_empty() { self.active_document_id = None; @@ -309,51 +319,24 @@ impl MessageHandler> for Portfolio responses.add(PortfolioMessage::SelectDocument { document_id }); } } - PortfolioMessage::DocumentStorageMounted { document_id, reopened, mounted } => { - let Some(document) = self.documents.get_mut(&document_id) else { - // Document was closed before its working copy finished mounting. - return; - }; - let Some(mounted) = mounted else { - log::error!("DocumentStorageMounted for {document_id:?} arrived without its payload"); - return; - }; - let (gdd, declarations) = *mounted; - document.set_storage(gdd, declarations); - if !reopened { - document.commit_storage_snapshot(&resource_storage.resources_mut(), preferences.validate_storage_round_trip); - document.retire_storage_interaction(); - } - } PortfolioMessage::DestroyAllDocuments => { // Empty the list of internal document data self.documents.clear(); - self.unloaded_documents.clear(); + self.loading_documents.clear(); self.document_ids.clear(); self.active_document_id = None; + self.restore_select = None; responses.add(MenuBarMessage::SendLayout); responses.add(PersistentStateMessage::WriteState); } PortfolioMessage::EditorPreferences => self.executor.update_editor_preferences(preferences.editor_preferences()), PortfolioMessage::GarbageCollectResources => { - if !self.persistent_state.loaded() { - // We don't know what can be safely garbage collected - return; - } - if self.pending_opens > 0 { - // A document not fully loaded, skip garbage collection. + // The resources of documents that are not loaded yet are unknown, so GC waits for every load and for a successful store listing. + if !self.persistent_state.loaded() || self.pending_opens > 0 || !self.loading_documents.is_empty() || self.store_unavailable { return; } let mut used_resources = HashSet::new(); - for (id, info) in self.unloaded_documents.iter() { - if let Some(resources) = &info.resources { - used_resources.extend(resources.iter()); - } else { - responses.add(PersistentStateMessage::ReadDocument { document_id: *id }); - return; - } - } for document in self.documents.values_mut() { document.garbage_collect_resources(); used_resources.extend(document.resources.registry.resolved().filter_map(|info| info.hash.cloned())); @@ -397,70 +380,97 @@ impl MessageHandler> for Portfolio workspace_layout: _, } = state; - // TODO: Eventually remove this document upgrade code - let mut newly_unloaded_ids = Vec::new(); - for info in documents { if !self.document_ids.contains(&info.id) { self.document_ids.push_back(info.id); - } - if !self.documents.contains_key(&info.id) && !self.unloaded_documents.contains_key(&info.id) { - // TODO: Eventually remove this document upgrade code - newly_unloaded_ids.push(info.id); - - self.unloaded_documents.insert(info.id, info); + self.loading_documents.insert(info.id, info); } } + self.restore_select = current_document; responses.add(PortfolioMessage::UpdateOpenDocumentsList); - - let select_document_id = current_document.filter(|id| self.document_ids.contains(id)).or_else(|| self.document_ids.front().copied()); - - // Eagerly load every autosaved doc on startup so deserialization failures can be reported in one batched dialog at the end. - // The active doc's read is deferred to the `SelectDocument` below, but is still counted. - // TODO: Eventually remove this document upgrade code - self.failed_documents.expect_autosave_loads(newly_unloaded_ids.len()); - - // TODO: Eventually remove this document upgrade code - for document_id in &newly_unloaded_ids { - if Some(*document_id) != select_document_id { - self.pending_eager_loads.insert(*document_id); - responses.add(PersistentStateMessage::ReadDocument { document_id: *document_id }); + responses.add(list_stored_documents(self.document_store.clone())); + } + PortfolioMessage::StoredDocumentsListed { document_ids } => { + let Some(stored) = document_ids else { + self.store_unavailable = true; + Self::storage_error("The stored documents could not be listed, so they cannot be opened.".to_string(), responses); + return; + }; + let stored: HashSet<_> = stored.into_iter().collect(); + + // Session entries without stored content are dropped; stored documents the session does not know are appended. + self.document_ids.retain(|id| stored.contains(id) || self.documents.contains_key(id)); + self.loading_documents.retain(|id, _| stored.contains(id)); + for document_id in stored { + if !self.document_ids.contains(&document_id) { + self.document_ids.push_back(document_id); + self.loading_documents.insert( + document_id, + DocumentInfo { + id: document_id, + name: String::new(), + path: None, + is_saved: false, + }, + ); } } - if let Some(document_id) = select_document_id { - responses.add(PortfolioMessage::SelectDocument { document_id }); - } - // TODO: Eventually remove this document upgrade code - else if self.failed_documents.has_failures_to_report() { - responses.add(FailedDocumentsMessage::ShowFailedToLoadDocumentsDialog); + for document_id in self.loading_documents.keys().copied().collect::>() { + self.pending_opens += 1; + responses.add(load_stored_document( + self.document_store.clone(), + document_id, + resource_storage.resources_mut(), + reset_node_definitions_on_open, + !preferences.save_as_gdd, + )); } + responses.add(PortfolioMessage::UpdateOpenDocumentsList); } - PortfolioMessage::LoadDocumentContent { + PortfolioMessage::StoredDocumentLoaded { document_id, - document_serialized_content, + document, + gdd, + declarations, } => { - let Some(info) = self.unloaded_documents.remove(&document_id) else { - log::error!("Tried to load content for non existent document"); + self.pending_opens = self.pending_opens.saturating_sub(1); + let Some(info) = self.loading_documents.remove(&document_id) else { + // Closed while loading. + responses.add(remove_stored_document(self.document_store.clone(), document_id)); return; }; - responses.add(PortfolioMessage::LoadDocument { - document_id, - document_name: Some(info.name), - document_path: info.path, - document_is_auto_saved: true, - document_is_saved: info.is_saved, - document_serialized_content, - }); + match document.map(|boxed| *boxed) { + Some(mut document) => { + // The legacy file carries no name, so a document the session does not know gets a fresh one. + document.name = if info.name.trim().is_empty() { + self.generate_new_document_name(Some(document_id)) + } else { + info.name + }; + document.path = info.path; + document.set_save_state(info.is_saved); + self.load_document(document, document_id, resource_storage.resources_mut(), !preferences.save_as_gdd, responses); + self.attach_storage(document_id, gdd, declarations, resource_storage, preferences); + } + None => { + self.document_ids.retain(|id| *id != document_id); + self.failed_documents.record_failure(document_id, info); + if self.active_document_id == Some(document_id) { + self.active_document_id = None; + } + } + } - // Suppress auto-select for startup eager loads to keep focus on the user's active doc - // TODO: Eventually remove this document upgrade code - // TODO: (But keep the inner logic unconditionally, just remove the condition) - if !self.pending_eager_loads.remove(&document_id) { - responses.add(PortfolioMessage::SelectDocument { document_id }); + if self.active_document().is_none() { + self.select_after_load(responses); } + if self.pending_opens == 0 { + responses.add(FailedDocumentsMessage::ShowFailedToLoadDocumentsDialog); + } + responses.add(PortfolioMessage::UpdateOpenDocumentsList); } PortfolioMessage::NewDocumentWithName { name } => { let mut new_document = DocumentMessageHandler::default(); @@ -474,35 +484,33 @@ impl MessageHandler> for Portfolio responses.add(NavigationMessage::CanvasPan { delta: (0., 0.).into() }); } - self.load_document(new_document, document_id, resource_storage, preferences.validate_storage_round_trip, responses); + self.load_document(new_document, document_id, resource_storage.resources_mut(), !preferences.save_as_gdd, responses); responses.add(PortfolioMessage::SelectDocument { document_id }); } - PortfolioMessage::NextDocument => { - if let Some(active_document_id) = self.active_document_id { - let current_index = self.document_index(active_document_id); - let next_index = (current_index + 1) % self.document_ids.len(); - let next_id = self.document_ids[next_index]; - - responses.add(PortfolioMessage::SelectDocument { document_id: next_id }); - } - } - PortfolioMessage::OpenDocumentFile { + PortfolioMessage::OpenLegacyDocumentFile { document_name, document_path, document_serialized_content, } => { + let mut document = match load_legacy_document(document_serialized_content, &resource_storage.resources_mut(), reset_node_definitions_on_open) { + Ok(document) => document, + Err(error) => { + log::error!("{error}"); + let document_name = opened_document_name(document_name, document_path.as_deref()); + simple_dialogs::FailedToOpenDocumentDialog { document_name }.send_dialog_to_frontend(responses); + return; + } + }; let document_id = DocumentId(generate_uuid()); - responses.add(PortfolioMessage::LoadDocument { - document_id, - document_name, - document_path, - document_is_auto_saved: false, - document_is_saved: true, - document_serialized_content, - }); + document.set_save_state(true); + document.name = self.resolve_document_name(opened_document_name(document_name, document_path.as_deref()), None); + document.path = document_path; + + self.load_document(document, document_id, resource_storage.resources_mut(), !preferences.save_as_gdd, responses); responses.add(PortfolioMessage::SelectDocument { document_id }); + responses.add(AppWindowMessage::Focus); } - PortfolioMessage::OpenGddDocument { + PortfolioMessage::OpenDocumentFile { document_name, document_path, content, @@ -512,203 +520,85 @@ impl MessageHandler> for Portfolio // Suppress resource GC until the open completes self.pending_opens += 1; - responses.add(open_gdd_document( - self.working_copy_root.clone(), + responses.add(open_document_file( + self.document_store.clone(), document_id, document_name, document_path, content, resource_storage.resources_mut(), - preferences.validate_storage_round_trip, + reset_node_definitions_on_open, + !preferences.save_as_gdd, )); } - PortfolioMessage::GddDocumentLoaded { + PortfolioMessage::DocumentFileLoaded { document_id, document_name, document_path, document, + gdd, + declarations, } => { self.pending_opens = self.pending_opens.saturating_sub(1); let Some(mut document) = document.map(|boxed| *boxed) else { - let name = document_name - .filter(|name| !name.trim().is_empty()) - .or_else(|| document_path.as_ref().and_then(|path| path.file_stem()).map(|stem| stem.to_string_lossy().into_owned())) - .unwrap_or_default(); - simple_dialogs::FailedToOpenDocumentDialog { document_name: name }.send_dialog_to_frontend(responses); + let document_name = opened_document_name(document_name, document_path.as_deref()); + simple_dialogs::FailedToOpenDocumentDialog { document_name }.send_dialog_to_frontend(responses); + responses.add(remove_stored_document(self.document_store.clone(), document_id)); return; }; - document.finalize_storage_load(); document.set_save_state(true); - - let name = document_name - .filter(|name| !name.trim().is_empty()) - .or_else(|| document_path.as_ref().and_then(|path| path.file_stem()).map(|stem| stem.to_string_lossy().into_owned())) - .unwrap_or_else(|| DEFAULT_DOCUMENT_NAME.to_string()); - document.name = self.resolve_document_name(name, None); + document.name = self.resolve_document_name(opened_document_name(document_name, document_path.as_deref()), None); document.path = document_path; - // The working copy is already mounted (we opened the .gdd), so skip the async re-mount. - self.load_document(document, document_id, resource_storage, preferences.validate_storage_round_trip, responses); + self.load_document(document, document_id, resource_storage.resources_mut(), !preferences.save_as_gdd, responses); + self.attach_storage(document_id, gdd, declarations, resource_storage, preferences); responses.add(PortfolioMessage::SelectDocument { document_id }); } - PortfolioMessage::ToggleResetNodesToDefinitionsOnOpen => { - self.reset_node_definitions_on_open = !self.reset_node_definitions_on_open; - responses.add(MenuBarMessage::SendLayout); + PortfolioMessage::StorageUpdated => { + for (&document_id, document) in &mut self.documents { + match (preferences.save_as_gdd, document.storage().is_some(), document.container.clone()) { + (true, false, Some(container)) => responses.add(open_storage(container, document_id, resource_storage.resources_mut())), + (false, true, _) => document.clear_storage(), + _ => {} + } + } } - PortfolioMessage::LoadDocument { + PortfolioMessage::StorageAttached { document_id, - document_name, - document_path, - document_is_auto_saved, - document_is_saved, - document_serialized_content, + container, + gdd, + declarations, } => { - // Upgrade the document being opened to use fresh copies of all nodes - let reset_node_definitions_on_open = reset_node_definitions_on_open || document_migration_reset_node_definition(&document_serialized_content); - // Upgrade the document being opened with string replacements on the original JSON - let document_serialized_content = document_migration_string_preprocessing(document_serialized_content); - // Upgrade resources from being referend by hash to beeing referened by ID - let (document_serialized_content, resource_hash_to_id_migration_map) = document_migration_replace_resources_referenced_by_hash(document_serialized_content); - - // Deserialize the document - let document = DocumentMessageHandler::deserialize_document(&document_serialized_content); - - // Display an error to the user if the document could not be opened - let mut document = match document { - Ok(document) => document, - Err(e) => { - log::error!("{e}"); - // TODO: Eventually remove this document upgrade code - // TODO: (Only the `if` branch, the `else` branch's manual-open dialog stays) - if document_is_auto_saved { - let name = document_name.unwrap_or_default(); - let info = DocumentInfo { - id: document_id, - name, - resources: None, - path: document_path, - is_saved: document_is_saved, - }; - self.document_ids.retain(|id| id != &document_id); - self.failed_documents.record_failure(document_id, info, document_serialized_content); - - if self.active_document_id == Some(document_id) { - self.active_document_id = None; - if let Some(next_id) = self.document_ids.front().copied() { - responses.add(PortfolioMessage::SelectDocument { document_id: next_id }); - } - } - - responses.add(PortfolioMessage::UpdateOpenDocumentsList); - self.failed_documents.tick_autosave_load_progress(responses, true); - } else { - let name = document_name - .filter(|n| !n.trim().is_empty()) - .or_else(|| document_path.as_ref().and_then(|p| p.file_stem()).map(|s| s.to_string_lossy().into_owned())) - .unwrap_or_default(); - let dialog = simple_dialogs::FailedToOpenDocumentDialog { document_name: name }; - dialog.send_dialog_to_frontend(responses); - } - - return; - } + let Some(document) = self.documents.get_mut(&document_id) else { + // Closed while its storage was opening. + responses.add(remove_stored_document(self.document_store.clone(), document_id)); + return; }; - - // Upgrade the document's nodes to be compatible with the latest version - document_migration_upgrades(&mut document, reset_node_definitions_on_open); - - // Load the document's embedded resources into the resource storage - std::mem::take(&mut document.resources.embedded).into_iter().for_each(|(hash, resource)| { - let data: Arc<[u8]> = Arc::from(resource.as_ref()); - if ResourceHash::from(data.as_ref()) != hash { - log::error!("Resource hash mismatch for resource with hash {hash}"); - return; - } - responses.add(ResourceStorageMessage::Store { data }); - - // TODO: Eventually remove this document upgrade code - // Register any resources that were previously referenced by hash - if let Some(id) = resource_hash_to_id_migration_map.get(&hash) - && !document.resources.registry.contains(id) - { - document.resources.registry.resolve(id, hash); - document.resources.registry.push_source_back(id, DataSource::Embedded); - } - }); - - // Ensure each node has the metadata for its inputs - for (node_id, node, path) in document.network_interface.document_network().clone().recursive_nodes() { - document.network_interface.validate_input_metadata(node_id, node, &path); - document.network_interface.validate_output_names(node_id, node, &path); - } - // Ensure layers are positioned as stacks if they are upstream siblings of another layer - document.network_interface.load_structure(); - let all_layers = LayerNodeIdentifier::ROOT_PARENT.descendants(document.network_interface.document_metadata()).collect::>(); - for layer in all_layers { - let Some((downstream_node, input_index)) = document - .network_interface - .outward_wires(&[]) - .and_then(|outward_wires| outward_wires.get(&OutputConnector::primary_output(layer.to_node()))) - .and_then(|outward_wires| outward_wires.first()) - .and_then(|input_connector| input_connector.node_id().map(|node_id| (node_id, input_connector.input_index()))) - else { - continue; - }; - - // If the downstream node is a layer and the input is the first input and the current layer is not in a stack - if input_index == 0 && document.network_interface.is_layer(&downstream_node, &[]) && !document.network_interface.is_stack(&layer.to_node(), &[]) { - // Ensure the layer is horizontally aligned with the downstream layer to prevent changing the layout of old files - let (Some(layer_position), Some(downstream_position)) = - (document.network_interface.position(&layer.to_node(), &[]), document.network_interface.position(&downstream_node, &[])) - else { - log::error!("Could not get position for layer {:?} or downstream node {} when opening file", layer.to_node(), downstream_node); - continue; - }; - - if layer_position.x == downstream_position.x { - document.network_interface.set_stack_position_calculated_offset(&layer.to_node(), &downstream_node, &[]); - } - } - } - - // Everything above is how the document arrives rather than an edit to it, so what those fix-ups - // recorded must not ride along in the first real commit. They still have to reach storage, so - // the next commit converts the whole document instead of staging the batch that follows them. - document.network_interface.discard_deltas(); - document.require_whole_document_stage(); - - // Set the save state of the document based on what's given to us by the caller to this message - document.set_auto_save_state(document_is_auto_saved); - document.set_save_state(document_is_saved); - - let document_name_from_path = document_path.as_ref().and_then(|path| { - if path.extension().is_some_and(|extension| extension == FILE_EXTENSION || extension == GDD_FILE_EXTENSION) { - path.file_stem().map(|stem| stem.to_string_lossy().to_string()) - } else { - None - } - }); - - let candidate_name = match (document_name, document_path, document_name_from_path) { - (Some(name), _, None) => name, - (_, Some(path), Some(name)) => { - document.path = Some(path); - name - } - (_, _, Some(name)) => name, - _ => String::new(), + let Some(container) = container else { + Self::storage_error("The document's storage could not be opened, so it will not be autosaved.".to_string(), responses); + return; }; - document.name = self.resolve_document_name(candidate_name, None); - - // Load the document into the portfolio so it opens in the editor - self.load_document(document, document_id, resource_storage, preferences.validate_storage_round_trip, responses); - - responses.add(AppWindowMessage::Focus); + document.container = Some(container); + self.attach_storage(document_id, gdd, declarations, resource_storage, preferences); + self.save_if_needed(document_id, responses); + } + PortfolioMessage::NextDocument => { + if let Some(active_document_id) = self.active_document_id { + let current_index = self.document_index(active_document_id); + let next_index = (current_index + 1) % self.document_ids.len(); + let next_id = self.document_ids[next_index]; - // TODO: Eventually remove this document upgrade code - if document_is_auto_saved { - self.failed_documents.tick_autosave_load_progress(responses, false); + responses.add(PortfolioMessage::SelectDocument { document_id: next_id }); + } + } + PortfolioMessage::PrevDocument => { + if let Some(active_document_id) = self.active_document_id { + let len = self.document_ids.len(); + let current_index = self.document_index(active_document_id); + let prev_index = (current_index + len - 1) % len; + let prev_id = self.document_ids[prev_index]; + responses.add(PortfolioMessage::SelectDocument { document_id: prev_id }); } } PortfolioMessage::CenterLayers { layers } => { @@ -816,15 +706,6 @@ impl MessageHandler> for Portfolio responses.add(NodeGraphMessage::RunDocumentGraph); } } - PortfolioMessage::PrevDocument => { - if let Some(active_document_id) = self.active_document_id { - let len = self.document_ids.len(); - let current_index = self.document_index(active_document_id); - let prev_index = (current_index + len - 1) % len; - let prev_id = self.document_ids[prev_index]; - responses.add(PortfolioMessage::SelectDocument { document_id: prev_id }); - } - } PortfolioMessage::ReorderDocument { document_id, new_index } => { let new_index = new_index.min(self.document_ids.len().saturating_sub(1)); let Some(current_index) = self.document_ids.iter().position(|&id| id == document_id) else { @@ -899,10 +780,6 @@ impl MessageHandler> for Portfolio layout_target: LayoutTarget::StatusBarInfo, }); } - PortfolioMessage::RenameDocument { new_name } => { - let resolved_name = self.resolve_document_name(new_name, self.active_document_id); - responses.add(DocumentMessage::RenameDocument { new_name: resolved_name }); - } PortfolioMessage::SelectDocument { document_id } => { // Auto-save the document we are leaving let mut node_graph_open = false; @@ -916,18 +793,12 @@ impl MessageHandler> for Portfolio node_graph_open = document.is_graph_overlay_open(); } - if self.unloaded_documents.contains_key(&document_id) { - let already_selected = self.active_document_id == Some(document_id); - + if self.loading_documents.contains_key(&document_id) { self.active_document_id = Some(document_id); + self.restore_select = Some(document_id); responses.add(MenuBarMessage::SendLayout); responses.add(PortfolioMessage::UpdateOpenDocumentsList); responses.add(FrontendMessage::UpdateActiveDocument { document_id }); - - if !already_selected { - responses.add(PersistentStateMessage::ReadDocument { document_id }); - } - return; } @@ -938,6 +809,7 @@ impl MessageHandler> for Portfolio // Set the new active document ID self.active_document_id = Some(document_id); + self.restore_select = None; responses.add(MenuBarMessage::SendLayout); responses.add(PortfolioMessage::UpdateOpenDocumentsList); @@ -969,6 +841,10 @@ impl MessageHandler> for Portfolio responses.add(PropertiesPanelMessage::Clear); } } + PortfolioMessage::RenameDocument { new_name } => { + let resolved_name = self.resolve_document_name(new_name, self.active_document_id); + responses.add(DocumentMessage::RenameDocument { new_name: resolved_name }); + } PortfolioMessage::SubmitDocumentExport { name, file_type, @@ -1097,6 +973,10 @@ impl MessageHandler> for Portfolio Ok(message) => responses.add_front(message), } } + PortfolioMessage::ToggleResetNodesToDefinitionsOnOpen => { + self.reset_node_definitions_on_open = !self.reset_node_definitions_on_open; + responses.add(MenuBarMessage::SendLayout); + } PortfolioMessage::ToggleRulers => { if let Some(document) = self.active_document_mut() { document.rulers_visible = !document.rulers_visible; @@ -1153,8 +1033,13 @@ impl PortfolioMessageHandler { Self { executor, ..Default::default() } } - pub fn set_working_copy_root(&mut self, root: Option) { - self.working_copy_root = root; + pub fn set_document_store(&mut self, store: Arc) { + self.document_store = DocumentStoreHandle::new(store); + } + + #[cfg(test)] + pub(crate) fn document_store(&self) -> DocumentStoreHandle { + self.document_store.clone() } pub fn document(&self, document_id: DocumentId) -> Option<&DocumentMessageHandler> { @@ -1187,16 +1072,8 @@ impl PortfolioMessageHandler { } pub fn persisted_state_snapshot(&self) -> PersistedState { - let mut documents = self.document_ids.iter().filter_map(|id| self.document_details(*id)).collect::>(); - - // Keep failed-to-load docs referenced in `state.documents` so their autosave files survive `garbage_collect_document_files` - // TODO: Eventually remove this document upgrade code - for info in self.failed_documents.failed_document_infos() { - documents.push(info.clone()); - } - PersistedState { - documents, + documents: self.document_ids.iter().filter_map(|id| self.document_details(*id)).collect(), current_document: self.active_document_id, workspace_layout: Some(self.workspace.panel_layout.clone()), } @@ -1235,19 +1112,12 @@ impl PortfolioMessageHandler { if new_number == 1 { untitled.to_string() } else { format!("{untitled} {new_number}") } } - fn load_document( - &mut self, - mut new_document: DocumentMessageHandler, - document_id: DocumentId, - resource_storage: &ResourceStorageMessageHandler, - validate: bool, - responses: &mut VecDeque, - ) { + fn load_document(&mut self, mut new_document: DocumentMessageHandler, document_id: DocumentId, resources: ResourcesHandle, legacy_only: bool, responses: &mut VecDeque) { let is_new_document = !self.document_ids.contains(&document_id); if is_new_document { self.document_ids.push_back(document_id); } - self.unloaded_documents.remove(&document_id); + self.loading_documents.remove(&document_id); new_document.update_layers_panel_control_bar_widgets( self.workspace.panel_layout.is_panel_visible(PanelType::Layers) && !self.workspace.panel_layout.focus_document, responses, @@ -1270,67 +1140,60 @@ impl PortfolioMessageHandler { responses.add(PortfolioMessage::UpdateOpenDocumentsList); } - // Mount the per-document `Gdd` working copy asynchronously. - // - // Only mount for legacy loads: a `.gdd` open already mounted its working copy - // and remounting would overwrite it and drop that history state. - if self.documents.get(&document_id).and_then(|document| document.storage()).is_none() { - let legacy_network = self - .documents - .get(&document_id) - .map(|document| document.network_interface.document_network().clone()) - .unwrap_or_default(); - - responses.add(Self::mount_document_storage( - self.working_copy_root.clone(), - document_id, - legacy_network, - resource_storage.resources(), - validate, - )); + if self.documents[&document_id].container.is_none() { + responses.add(attach_container(self.document_store.clone(), document_id, resources, legacy_only)); + } else { + self.save_if_needed(document_id, responses); } } - /// With a configured root the working copy lives at `/`; without one it is in-memory. - /// - /// On a *reopen* (existing working copy), the freshly-read `.gdd` is converted back to a runtime - /// network and compared against `legacy_network` before - /// the working copy attaches. It validates the `.gdd` *read* path. - fn mount_document_storage( - working_copy_root: Option, + /// Installs the storage session and stages the current network so a session that fell behind the legacy + /// document catches up. Identical content stages nothing. + fn attach_storage( + &mut self, document_id: DocumentId, - legacy_network: graph_craft::document::NodeNetwork, - byte_store: Box, - validate: bool, - ) -> Message { - let path = working_copy_root.map(|root| root.join(format!("{:016x}", document_id.0))); - let editor_version = crate::application::GRAPHITE_GIT_COMMIT_HASH.to_string(); - let peer = document_graph_storage::PeerId(generate_uuid()); - - let future = async move { - let (gdd, reopened) = match build_or_open_working_copy(path.as_deref(), peer, document_id.0, editor_version).await { - Ok(result) => result, - Err(error) => { - log::error!("Failed to mount document storage for {document_id:?}: {error}"); - return Message::NoOp; - } - }; - - let declarations = gdd.declarations(byte_store.as_ref()).await; + gdd: Option>, + declarations: Declarations, + resource_storage: &ResourceStorageMessageHandler, + preferences: &PreferencesMessageHandler, + ) { + let (Some(document), Some(gdd)) = (self.documents.get_mut(&document_id), gdd) else { return }; + document.set_storage(*gdd, declarations); + document.require_whole_document_stage(); + document.commit_storage_snapshot(&resource_storage.resources_mut(), preferences.validate_storage_round_trip); + document.retire_storage_interaction(); + } - if validate && reopened { - compare_storage_against_runtime(&gdd, &legacy_network, &declarations, document_id); - } + fn save_if_needed(&self, document_id: DocumentId, responses: &mut VecDeque) { + if let Some(document) = self.documents.get(&document_id) + && document.container.is_some() + && !document.is_auto_saved() + { + responses.add(PortfolioMessage::AutoSaveDocument { document_id }); + } + } - Message::Portfolio(PortfolioMessage::DocumentStorageMounted { - document_id, - reopened, - mounted: Some(Box::new((gdd, declarations))), - }) + /// Selects the restored document once it is loaded, or the first document when it failed or the session named none. + fn select_after_load(&mut self, responses: &mut VecDeque) { + let target = self.restore_select.filter(|id| self.document_ids.contains(id)).or_else(|| self.document_ids.front().copied()); + let Some(target) = target else { + self.restore_select = None; + return; }; - future.into() + if self.documents.contains_key(&target) { + self.restore_select = None; + responses.add(PortfolioMessage::SelectDocument { document_id: target }); + } else { + self.restore_select = Some(target); + } } + fn storage_error(description: String, responses: &mut VecDeque) { + responses.add(DialogMessage::DisplayDialogError { + title: "Document storage failed".to_string(), + description, + }); + } /// Returns an iterator over the open documents in order. pub fn ordered_document_iterator(&self) -> impl Iterator { self.document_ids.iter().filter_map(|id| self.document(*id)) @@ -1371,10 +1234,9 @@ impl PortfolioMessageHandler { name: document.name.clone(), path: document.path.clone(), is_saved: document.is_saved(), - resources: Some(document.resources.registry.resolved().filter_map(|info| info.hash.cloned()).collect::>().into_boxed_slice()), }) } else { - self.unloaded_documents.get(&document_id).cloned() + self.loading_documents.get(&document_id).cloned() } } @@ -1405,3 +1267,10 @@ impl PortfolioMessageHandler { path } } + +/// The name an opened file gets: the given name, else the file stem of its path, else empty so the portfolio generates one. +fn opened_document_name(name: Option, path: Option<&Path>) -> String { + name.filter(|name| !name.trim().is_empty()) + .or_else(|| path.and_then(Path::file_stem).map(|stem| stem.to_string_lossy().into_owned())) + .unwrap_or_default() +} diff --git a/editor/src/messages/preferences/preferences_message_handler.rs b/editor/src/messages/preferences/preferences_message_handler.rs index 58bfcc50355..48e6ca272de 100644 --- a/editor/src/messages/preferences/preferences_message_handler.rs +++ b/editor/src/messages/preferences/preferences_message_handler.rs @@ -94,6 +94,7 @@ impl MessageHandler> for Prefe }); responses.add(FrontendMessage::UpdateUIScale { scale: self.ui_scale }); responses.add(ToolMessage::RefreshToolShelf); + responses.add(PortfolioMessage::StorageUpdated); } PreferencesMessage::ResetToDefaults => { responses.add(PreferencesMessage::Load { preferences: Self::default() }); @@ -144,6 +145,7 @@ impl MessageHandler> for Prefe } PreferencesMessage::SaveAsGdd { enabled } => { self.save_as_gdd = enabled; + responses.add(PortfolioMessage::StorageUpdated); } PreferencesMessage::ToggleShowStoragePreferences => { self.show_storage_preferences = !self.show_storage_preferences; diff --git a/editor/src/test_utils.rs b/editor/src/test_utils.rs index f8888d659fb..d17b0a3310f 100644 --- a/editor/src/test_utils.rs +++ b/editor/src/test_utils.rs @@ -27,8 +27,8 @@ impl EditorTestUtils { editor.handle_message(PortfolioMessage::Init); - // Enable the dual-write soak validation in tests so storage round-trip drift fails loud, matching - // the previous always-on-in-debug behavior now that it is gated by a preference. + // Tests run with `.gdd` storage on and the dual-write soak validation enabled, so storage round-trip drift fails loud. + editor.handle_message(PreferencesMessage::SaveAsGdd { enabled: true }); editor.handle_message(PreferencesMessage::ValidateStorageRoundTrip { enabled: true }); Self { editor, runtime } diff --git a/frontend/src/managers/persistence.ts b/frontend/src/managers/persistence.ts index 87c598a5f01..3298f07110e 100644 --- a/frontend/src/managers/persistence.ts +++ b/frontend/src/managers/persistence.ts @@ -1,14 +1,6 @@ import type { PortfolioStore } from "/src/stores/portfolio"; import type { SubscriptionsRouter } from "/src/subscriptions-router"; -import { - saveEditorPreferences, - loadEditorPreferences, - writePersistedState, - readPersistedState, - writePersistedDocument, - readPersistedDocument, - deletePersistedDocument, -} from "/src/utility-functions/persistence"; +import { saveEditorPreferences, loadEditorPreferences, writePersistedState, readPersistedState } from "/src/utility-functions/persistence"; import type { EditorWrapper } from "/wrapper/pkg/graphite_wasm_wrapper"; let subscriptionsRouter: SubscriptionsRouter | undefined = undefined; @@ -38,18 +30,6 @@ export function createPersistenceManager(subscriptions: SubscriptionsRouter, edi await readPersistedState(editor); }); - subscriptions.subscribeFrontendMessage("TriggerPersistenceWriteDocument", async (data) => { - await writePersistedDocument(data); - }); - - subscriptions.subscribeFrontendMessage("TriggerPersistenceReadDocument", async (data) => { - await readPersistedDocument(data.documentId, editor); - }); - - subscriptions.subscribeFrontendMessage("TriggerPersistenceDeleteDocument", async (data) => { - await deletePersistedDocument(String(data.documentId)); - }); - subscriptions.subscribeFrontendMessage("TriggerOpenLaunchDocuments", async () => { // TODO: Could be used to load documents from URL params or similar on launch }); @@ -63,9 +43,6 @@ export function destroyPersistenceManager() { subscriptions.unsubscribeFrontendMessage("TriggerLoadPreferences"); subscriptions.unsubscribeFrontendMessage("TriggerPersistenceWriteState"); subscriptions.unsubscribeFrontendMessage("TriggerPersistenceReadState"); - subscriptions.unsubscribeFrontendMessage("TriggerPersistenceWriteDocument"); - subscriptions.unsubscribeFrontendMessage("TriggerPersistenceReadDocument"); - subscriptions.unsubscribeFrontendMessage("TriggerPersistenceDeleteDocument"); subscriptions.unsubscribeFrontendMessage("TriggerOpenLaunchDocuments"); } diff --git a/frontend/src/utility-functions/persistence.ts b/frontend/src/utility-functions/persistence.ts index c591753ba69..ef09c8268f3 100644 --- a/frontend/src/utility-functions/persistence.ts +++ b/frontend/src/utility-functions/persistence.ts @@ -1,4 +1,3 @@ -import type { MessageBody } from "/src/subscriptions-router"; import type { DocumentInfo, EditorWrapper, PersistedState } from "/wrapper/pkg/graphite_wasm_wrapper"; const PERSISTENCE_DB = "graphite"; @@ -33,39 +32,9 @@ function reorderDocuments(documents: DocumentInfo[], orderedIds: bigint[]): Docu return reordered; } -// ==================================== -// State-based persistence (new format) -// ==================================== - -export async function writePersistedDocument(autoSaveDocument: MessageBody<"TriggerPersistenceWriteDocument">) { - const { documentId, document } = autoSaveDocument; - - // Update content in the documents store - await databaseUpdate>("documents", (old) => { - const documents = old || {}; - documents[String(documentId)] = document; - return documents; - }); -} - -export async function readPersistedDocument(documentId: bigint, editor: EditorWrapper) { - const documentContents = await databaseGet>("documents"); - if (!documentContents) return; - - const content = documentContents[String(documentId)]; - if (content === undefined) return; - - editor.loadDocumentContent(documentId, content); -} - -export async function deletePersistedDocument(id: string) { - // Remove content from the documents store - await databaseUpdate>("documents", (old) => { - const documents = old || {}; - delete documents[id]; - return documents; - }); -} +// ======================= +// State-based persistence +// ======================= export async function writePersistedState(state: PersistedState) { // Keep state ordered and normalized before writing. @@ -74,16 +43,14 @@ export async function writePersistedState(state: PersistedState) { state.documents.map((entry) => entry.id), ); await databaseSet("state", state); - await garbageCollectDocuments(); } export async function readPersistedState(editor: EditorWrapper) { await migrateToNewFormat(); - await garbageCollectDocuments(); + await migrateDocumentsToStore(); const state = await databaseGet("state"); - if (!state) return; - editor.loadPersistedState(state); + editor.loadPersistedState(state ?? emptyPersistedState()); } export async function saveEditorPreferences(preferences: unknown) { @@ -95,36 +62,80 @@ export async function loadEditorPreferences(editor: EditorWrapper) { editor.loadPreferences(preferences ? JSON.stringify(preferences) : undefined); } -// Remove orphaned entries from the "documents" content store that have no corresponding entry in "state" -async function garbageCollectDocuments() { - const state = await databaseGet("state"); - const documentContents = await databaseGet>("documents"); - if (!documentContents) return; - - const validIds = new Set(state ? state.documents.map((doc) => String(doc.id)) : []); - let changed = false; +// ========================= +// Migration from old format +// ========================= - Object.keys(documentContents).forEach((key) => { - if (!validIds.has(key)) { - delete documentContents[key]; - changed = true; +// TODO: Eventually remove this document upgrade code +async function migrateDocumentsToStore() { + const documents = await databaseGet>("documents"); + if (!documents || Object.keys(documents).length === 0) return; + + const root = await navigator.storage.getDirectory(); + const store = await root.getDirectoryHandle("documents", { create: true }); + for (const [id, content] of Object.entries(documents)) { + const directory = await store.getDirectoryHandle(BigInt(id).toString(16).padStart(16, "0"), { create: true }); + if (!(await fileExists(directory, "legacy.graphite")) && !(await fileExists(directory, "manifest.json"))) { + const file = await directory.getFileHandle("legacy.graphite", { create: true }); + const writable = await file.createWritable(); + await writable.write(content); + await writable.close(); } + await markDocumentMigrated(id, content); + } +} + +async function fileExists(directory: FileSystemDirectoryHandle, name: string): Promise { + try { + await directory.getFileHandle(name); + return true; + } catch (error) { + if (error instanceof DOMException && error.name === "NotFoundError") return false; + throw error; + } +} + +// TODO: Eventually remove this document upgrade code +async function markDocumentMigrated(id: string, content: string) { + const db = await databaseOpen(); + await new Promise((resolve, reject) => { + const transaction = db.transaction(PERSISTENCE_STORE, "readwrite"); + const store = transaction.objectStore(PERSISTENCE_STORE); + const backupsRequest = store.get("documents_migrated"); + backupsRequest.onsuccess = () => { + const backups: Record = backupsRequest.result || {}; + backups[id] = content; + store.put(backups, "documents_migrated"); + const documentsRequest = store.get("documents"); + documentsRequest.onsuccess = () => { + const remaining: Record = documentsRequest.result || {}; + delete remaining[id]; + store.put(remaining, "documents"); + }; + }; + transaction.oncomplete = () => resolve(); + transaction.onerror = () => reject(transaction.error); }); +} - if (changed) await databaseSet("documents", documentContents); +async function wipeStoredDocuments() { + try { + const root = await navigator.storage.getDirectory(); + await root.removeEntry("documents", { recursive: true }); + } catch { + // Nothing stored, or OPFS is unavailable + } } export async function wipeDocuments() { await databaseDelete("state"); await databaseDelete("documents"); + await databaseDelete("documents_migrated"); + await wipeStoredDocuments(); await wipeOldFormat(); } -// ========================= -// Migration from old format -// ========================= - // TODO: Eventually remove this document upgrade code async function wipeOldFormat() { await databaseDelete("documents_tab_order"); @@ -253,18 +264,3 @@ async function databaseDelete(key: string): Promise { transaction.onerror = () => reject(transaction.error); }); } - -async function databaseUpdate(key: string, updater: (existing: T | undefined) => T): Promise { - const db = await databaseOpen(); - return new Promise((resolve, reject) => { - const transaction = db.transaction(PERSISTENCE_STORE, "readwrite"); - const store = transaction.objectStore(PERSISTENCE_STORE); - const getRequest = store.get(key); - getRequest.onsuccess = () => { - const existing: T | undefined = getRequest.result; - store.put(updater(existing), key); - }; - transaction.oncomplete = () => resolve(); - transaction.onerror = () => reject(transaction.error); - }); -} diff --git a/frontend/wrapper/src/editor_commands.rs b/frontend/wrapper/src/editor_commands.rs index ceea7c05630..0ee5375a870 100644 --- a/frontend/wrapper/src/editor_commands.rs +++ b/frontend/wrapper/src/editor_commands.rs @@ -197,14 +197,6 @@ mod editor_commands { PreferencesMessage::Load { preferences }.into() } - fn load_document_content(document_id: u64, document: String) -> Message { - PersistentStateMessage::LoadDocument { - document_id: DocumentId(document_id), - document, - } - .into() - } - fn select_document(document_id: u64) -> Message { PortfolioMessage::SelectDocument { document_id: DocumentId(document_id) }.into() } diff --git a/frontend/wrapper/src/editor_wrapper.rs b/frontend/wrapper/src/editor_wrapper.rs index c2d238a645f..4126d6c8a5e 100644 --- a/frontend/wrapper/src/editor_wrapper.rs +++ b/frontend/wrapper/src/editor_wrapper.rs @@ -74,10 +74,8 @@ impl EditorWrapper { let application_io = PlatformApplicationIo::new().await; let wake = crate::helpers::async_wake_callback(); - // On web the working-copy root is an OPFS directory name (no real filesystem path); each - // document mounts under `documents/`. - let working_copy_root = Some(std::path::PathBuf::from("documents")); - let editor = Editor::new(Environment { platform: Platform::Web, host }, uuid_random_seed, storage, working_copy_root, application_io, wake); + let document_store = std::sync::Arc::new(editor::document_store::OpfsStore::new("documents")); + let editor = Editor::new(Environment { platform: Platform::Web, host }, uuid_random_seed, storage, document_store, application_io, wake); if EDITOR.with(|slot| slot.lock().ok().map(|mut guard| *guard = Some(editor))).is_none() { log::error!("Attempted to initialize the editor more than once");