From 5848c8c14745d519b78523af9948f631f6478a51 Mon Sep 17 00:00:00 2001 From: gwenn Date: Sun, 4 Jun 2023 19:08:49 +0200 Subject: [PATCH 1/2] Draft of serialize API --- Cargo.toml | 3 ++ src/error.rs | 1 - src/lib.rs | 3 ++ src/serialize.rs | 131 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 src/serialize.rs diff --git a/Cargo.toml b/Cargo.toml index 5a6c1dc..e63c337 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,8 @@ column_decltype = [] wasm32-wasi-vfs = ["libsqlite3-sys/wasm32-wasi-vfs"] # Note: doesn't support 32-bit. winsqlite3 = ["libsqlite3-sys/winsqlite3"] +# 3.23.0 +serialize = ["modern_sqlite"] # Helper feature for enabling most non-build-related optional features # or dependencies (except `session`). This is useful for running tests / clippy @@ -109,6 +111,7 @@ modern-full = [ ] bundled-full = ["modern-full", "bundled"] +default = ["serialize"] [dependencies] time = { version = "0.3.0", features = ["formatting", "macros", "parsing"], optional = true } diff --git a/src/error.rs b/src/error.rs index 797a216..2e46374 100644 --- a/src/error.rs +++ b/src/error.rs @@ -393,7 +393,6 @@ impl Error { #[cold] pub fn error_from_sqlite_code(code: c_int, message: Option) -> Error { - // TODO sqlite3_error_offset // 3.38.0, #1130 Error::SqliteFailure(ffi::Error::new(code), message) } diff --git a/src/lib.rs b/src/lib.rs index 11e1ad4..d417e3a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -119,6 +119,9 @@ mod params; mod pragma; mod raw_statement; mod row; +#[cfg(feature = "serialize")] +#[cfg_attr(docsrs, doc(cfg(feature = "serialize")))] +pub mod serialize; #[cfg(feature = "session")] #[cfg_attr(docsrs, doc(cfg(feature = "session")))] pub mod session; diff --git a/src/serialize.rs b/src/serialize.rs new file mode 100644 index 0000000..6608595 --- /dev/null +++ b/src/serialize.rs @@ -0,0 +1,131 @@ +//! Serialize a database. +use std::convert::TryInto; +use std::marker::PhantomData; +use std::ops::Deref; +use std::ptr::NonNull; + +use crate::error::error_from_handle; +use crate::ffi; +use crate::{Connection, DatabaseName, Result}; + +/// Shared serialized database +pub struct SharedData<'conn> { + phantom: PhantomData<&'conn Connection>, + ptr: NonNull, + sz: usize, +} + +/// Owned serialized database +pub struct OwnedData { + ptr: NonNull, + sz: usize, +} + +/// Serialized database +pub enum Data<'conn> { + /// Shared serialized database + Shared(SharedData<'conn>), + /// Owned serialized database + Owned(OwnedData), +} + +impl<'conn> Deref for Data<'conn> { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + let (ptr, sz) = match self { + Data::Owned(OwnedData { ptr, sz }) => (ptr.as_ptr(), *sz), + Data::Shared(SharedData { ptr, sz, .. }) => (ptr.as_ptr(), *sz), + }; + unsafe { std::slice::from_raw_parts(ptr, sz) } + } +} + +impl Drop for OwnedData { + fn drop(&mut self) { + unsafe { + ffi::sqlite3_free(self.ptr.as_ptr().cast()); + } + } +} + +impl Connection { + /// Serialize a database. + pub fn serialize<'conn>(&'conn self, schema: DatabaseName<'_>) -> Result> { + let schema = schema.as_cstring()?; + let mut sz = 0; + let mut ptr: *mut u8 = unsafe { + ffi::sqlite3_serialize( + self.handle(), + schema.as_ptr(), + &mut sz, + ffi::SQLITE_SERIALIZE_NOCOPY, + ) + }; + Ok(if ptr.is_null() { + ptr = unsafe { ffi::sqlite3_serialize(self.handle(), schema.as_ptr(), &mut sz, 0) }; + if ptr.is_null() { + return Err(unsafe { error_from_handle(self.handle(), ffi::SQLITE_NOMEM) }); + } + Data::Owned(OwnedData { + ptr: NonNull::new(ptr).unwrap(), + sz: sz.try_into().unwrap(), + }) + } else { + // shared buffer + Data::Shared(SharedData { + ptr: NonNull::new(ptr).unwrap(), + sz: sz.try_into().unwrap(), + phantom: PhantomData, + }) + }) + } + + /// Deserialize a database. + pub fn deserialize( + &mut self, + schema: DatabaseName<'_>, + data: Data<'_>, + read_only: bool, + ) -> Result<()> { + let schema = schema.as_cstring()?; + let (data, sz, flags) = match data { + Data::Owned(OwnedData { ptr, sz }) => ( + ptr.as_ptr(), // FIXME double-free => mem forget + sz.try_into().unwrap(), + if read_only { + ffi::SQLITE_DESERIALIZE_FREEONCLOSE | ffi::SQLITE_DESERIALIZE_READONLY + } else { + ffi::SQLITE_DESERIALIZE_FREEONCLOSE | ffi::SQLITE_DESERIALIZE_RESIZEABLE + }, + ), + Data::Shared(SharedData { ptr, sz, .. }) => ( + ptr.as_ptr(), // FIXME lifetime of ptr must be > lifetime self + sz.try_into().unwrap(), + if read_only { + ffi::SQLITE_DESERIALIZE_READONLY + } else { + 0 + }, + ), + }; + let rc = unsafe { + ffi::sqlite3_deserialize(self.handle(), schema.as_ptr(), data, sz, sz, flags) + }; + if rc != ffi::SQLITE_OK { + return Err(unsafe { error_from_handle(self.handle(), rc) }); + } + /* TODO + if let Some(mxSize) = mxSize { + unsafe { + ffi::sqlite3_file_control( + self.handle(), + schema.as_ptr(), + ffi::SQLITE_FCNTL_SIZE_LIMIT, + &mut mxSize, + ) + }; + }*/ + Ok(()) + } +} From 67d1e34eb4ec4a56e25022d5ce44c934a04e65d7 Mon Sep 17 00:00:00 2001 From: gwenn Date: Mon, 5 Jun 2023 19:56:23 +0200 Subject: [PATCH 2/2] Serialize and deserialize database --- Cargo.toml | 1 - src/serialize.rs | 91 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 61 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e63c337..9cdc6f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,7 +111,6 @@ modern-full = [ ] bundled-full = ["modern-full", "bundled"] -default = ["serialize"] [dependencies] time = { version = "0.3.0", features = ["formatting", "macros", "parsing"], optional = true } diff --git a/src/serialize.rs b/src/serialize.rs index 6608595..e482af4 100644 --- a/src/serialize.rs +++ b/src/serialize.rs @@ -8,7 +8,7 @@ use crate::error::error_from_handle; use crate::ffi; use crate::{Connection, DatabaseName, Result}; -/// Shared serialized database +/// Shared (SQLITE_SERIALIZE_NOCOPY) serialized database pub struct SharedData<'conn> { phantom: PhantomData<&'conn Connection>, ptr: NonNull, @@ -21,9 +21,31 @@ pub struct OwnedData { sz: usize, } +impl OwnedData { + /// SAFETY: Caller must be certain that `ptr` is allocated by + /// `sqlite3_malloc`. + pub unsafe fn from_raw_nonnull(ptr: NonNull, sz: usize) -> Self { + Self { ptr, sz } + } + + fn into_raw(self) -> (*mut u8, usize) { + let raw = (self.ptr.as_ptr(), self.sz); + std::mem::forget(self); + raw + } +} + +impl Drop for OwnedData { + fn drop(&mut self) { + unsafe { + ffi::sqlite3_free(self.ptr.as_ptr().cast()); + } + } +} + /// Serialized database pub enum Data<'conn> { - /// Shared serialized database + /// Shared (SQLITE_SERIALIZE_NOCOPY) serialized database Shared(SharedData<'conn>), /// Owned serialized database Owned(OwnedData), @@ -41,14 +63,6 @@ impl<'conn> Deref for Data<'conn> { } } -impl Drop for OwnedData { - fn drop(&mut self) { - unsafe { - ffi::sqlite3_free(self.ptr.as_ptr().cast()); - } - } -} - impl Connection { /// Serialize a database. pub fn serialize<'conn>(&'conn self, schema: DatabaseName<'_>) -> Result> { @@ -85,34 +99,22 @@ impl Connection { pub fn deserialize( &mut self, schema: DatabaseName<'_>, - data: Data<'_>, + data: OwnedData, read_only: bool, ) -> Result<()> { let schema = schema.as_cstring()?; - let (data, sz, flags) = match data { - Data::Owned(OwnedData { ptr, sz }) => ( - ptr.as_ptr(), // FIXME double-free => mem forget - sz.try_into().unwrap(), - if read_only { - ffi::SQLITE_DESERIALIZE_FREEONCLOSE | ffi::SQLITE_DESERIALIZE_READONLY - } else { - ffi::SQLITE_DESERIALIZE_FREEONCLOSE | ffi::SQLITE_DESERIALIZE_RESIZEABLE - }, - ), - Data::Shared(SharedData { ptr, sz, .. }) => ( - ptr.as_ptr(), // FIXME lifetime of ptr must be > lifetime self - sz.try_into().unwrap(), - if read_only { - ffi::SQLITE_DESERIALIZE_READONLY - } else { - 0 - }, - ), + let (data, sz) = data.into_raw(); + let sz = sz.try_into().unwrap(); + let flags = if read_only { + ffi::SQLITE_DESERIALIZE_FREEONCLOSE | ffi::SQLITE_DESERIALIZE_READONLY + } else { + ffi::SQLITE_DESERIALIZE_FREEONCLOSE | ffi::SQLITE_DESERIALIZE_RESIZEABLE }; let rc = unsafe { ffi::sqlite3_deserialize(self.handle(), schema.as_ptr(), data, sz, sz, flags) }; if rc != ffi::SQLITE_OK { + // TODO sqlite3_free(data) ? return Err(unsafe { error_from_handle(self.handle(), rc) }); } /* TODO @@ -129,3 +131,32 @@ impl Connection { Ok(()) } } + +#[cfg(test)] +mod test { + use super::*; + use crate::{Connection, DatabaseName, Result}; + + #[test] + fn serialize() -> Result<()> { + let db = Connection::open_in_memory()?; + db.execute_batch("CREATE TABLE x AS SELECT 'data'")?; + let data = db.serialize(DatabaseName::Main)?; + let Data::Owned(data) = data else { panic!("expected OwnedData")}; + assert!(data.sz > 0); + Ok(()) + } + + #[test] + fn deserialize() -> Result<()> { + let src = Connection::open_in_memory()?; + src.execute_batch("CREATE TABLE x AS SELECT 'data'")?; + let data = src.serialize(DatabaseName::Main)?; + let Data::Owned(data) = data else { panic!("expected OwnedData")}; + + let mut dst = Connection::open_in_memory()?; + dst.deserialize(DatabaseName::Main, data, false)?; + dst.execute("DELETE FROM x", [])?; + Ok(()) + } +}