Loadable extension

This commit is contained in:
gwenn 2023-07-09 10:39:21 +02:00
parent 5980013935
commit 1308cdaa9d
11 changed files with 14701 additions and 40 deletions

View File

@ -50,6 +50,7 @@ bundled-sqlcipher = ["libsqlite3-sys/bundled-sqlcipher", "bundled"]
bundled-sqlcipher-vendored-openssl = ["libsqlite3-sys/bundled-sqlcipher-vendored-openssl", "bundled-sqlcipher"]
buildtime_bindgen = ["libsqlite3-sys/buildtime_bindgen"]
limits = []
loadable_extension = ["libsqlite3-sys/loadable_extension"]
hooks = []
i128_blob = []
sqlcipher = ["libsqlite3-sys/sqlcipher"]

View File

@ -2,7 +2,7 @@
name = "libsqlite3-sys"
version = "0.26.0"
authors = ["The rusqlite developers"]
edition = "2018"
edition = "2021"
repository = "https://github.com/rusqlite/rusqlite"
description = "Native bindings to the libsqlite3 library"
license = "MIT"
@ -23,6 +23,7 @@ min_sqlite_version_3_14_0 = ["pkg-config", "vcpkg"]
# Bundle only the bindings file. Note that this does nothing if
# `buildtime_bindgen` is enabled.
bundled_bindings = []
loadable_extension = ["atomic", "prettyplease", "quote", "regex", "syn"]
# sqlite3_unlock_notify >= 3.6.12
unlock_notify = []
# 3.13.0
@ -42,9 +43,18 @@ winsqlite3 = []
[dependencies]
openssl-sys = { version = "0.9", optional = true }
atomic = { version = "0.5", optional = true }
[build-dependencies]
bindgen = { version = "0.66", optional = true, default-features = false, features = ["runtime"] }
pkg-config = { version = "0.3.19", optional = true }
cc = { version = "1.0", optional = true }
vcpkg = { version = "0.2", optional = true }
# for loadable_extension:
prettyplease = {version = "0.2", optional = true }
# like bindgen
quote = { version = "1", optional = true, default-features = false }
# like bindgen
regex = { version = "1.5", optional = true, default-features = false, features = ["std", "unicode"] }
# like bindgen
syn = { version = "2.0", optional = true, features = ["full", "extra-traits", "visit-mut"] }

File diff suppressed because it is too large Load Diff

View File

@ -28,8 +28,12 @@ fn is_compiler(compiler_name: &str) -> bool {
/// Copy bindgen file from `dir` to `out_path`.
fn copy_bindings<T: AsRef<Path>>(dir: &str, bindgen_name: &str, out_path: T) {
std::fs::copy(format!("{dir}/{bindgen_name}"), out_path)
.expect("Could not copy bindings to output directory");
let from = if cfg!(feature = "loadable_extension") {
format!("{dir}/{bindgen_name}_ext.rs")
} else {
format!("{dir}/{bindgen_name}.rs")
};
std::fs::copy(from, out_path).expect("Could not copy bindings to output directory");
}
fn main() {
@ -38,12 +42,14 @@ fn main() {
if cfg!(feature = "in_gecko") {
// When inside mozilla-central, we are included into the build with
// sqlite3.o directly, so we don't want to provide any linker arguments.
copy_bindings("sqlite3", "bindgen_bundled_version.rs", out_path);
copy_bindings("sqlite3", "bindgen_bundled_version", out_path);
return;
}
println!("cargo:rerun-if-env-changed=LIBSQLITE3_SYS_USE_PKG_CONFIG");
if env::var_os("LIBSQLITE3_SYS_USE_PKG_CONFIG").map_or(false, |s| s != "0") {
if env::var_os("LIBSQLITE3_SYS_USE_PKG_CONFIG").map_or(false, |s| s != "0")
|| cfg!(feature = "loadable_extension")
{
build_linked::main(&out_dir, &out_path);
} else if cfg!(all(
feature = "sqlcipher",
@ -106,7 +112,7 @@ mod build_bundled {
}
#[cfg(not(feature = "buildtime_bindgen"))]
{
super::copy_bindings(lib_name, "bindgen_bundled_version.rs", out_path);
super::copy_bindings(lib_name, "bindgen_bundled_version", out_path);
}
println!("cargo:rerun-if-changed={lib_name}/sqlite3.c");
println!("cargo:rerun-if-changed=sqlite3/wasm32-wasi-vfs.c");
@ -344,11 +350,28 @@ impl From<HeaderLocation> for String {
prefix, prefix
)
});
header.push_str("/sqlite3.h");
header.push_str(if cfg!(feature = "loadable_extension") {
"/sqlite3ext.h"
} else {
"/sqlite3.h"
});
header
}
HeaderLocation::Wrapper => "wrapper.h".into(),
HeaderLocation::FromPath(path) => format!("{}/sqlite3.h", path),
HeaderLocation::Wrapper => if cfg!(feature = "loadable_extension") {
"wrapper_ext.h"
} else {
"wrapper.h"
}
.into(),
HeaderLocation::FromPath(path) => format!(
"{}/{}",
path,
if cfg!(feature = "loadable_extension") {
"sqlite3ext.h"
} else {
"sqlite3.h"
}
),
}
}
}
@ -375,12 +398,13 @@ mod build_linked {
// on buildtime_bindgen instead, but this is still supported as we
// have runtime version checks and there are good reasons to not
// want to run bindgen.
super::copy_bindings(lib_name(), "bindgen_bundled_version.rs", out_path);
super::copy_bindings(lib_name(), "bindgen_bundled_version", out_path);
} else {
bindings::write_to_out_dir(header, out_path);
}
}
#[cfg(not(feature = "loadable_extension"))]
fn find_link_mode() -> &'static str {
// If the user specifies SQLITE3_STATIC (or SQLCIPHER_STATIC), do static
// linking, unless it's explicitly set to 0.
@ -404,9 +428,11 @@ mod build_linked {
// `links=` value in our Cargo.toml) to get this value. This might be
// useful if you need to ensure whatever crypto library sqlcipher relies
// on is available, for example.
#[cfg(not(feature = "loadable_extension"))]
println!("cargo:link-target={link_lib}");
if win_target() && cfg!(feature = "winsqlite3") {
#[cfg(not(feature = "loadable_extension"))]
println!("cargo:rustc-link-lib=dylib={link_lib}");
return HeaderLocation::Wrapper;
}
@ -416,6 +442,7 @@ mod build_linked {
// Try to use pkg-config to determine link commands
let pkgconfig_path = Path::new(&dir).join("pkgconfig");
env::set_var("PKG_CONFIG_PATH", pkgconfig_path);
#[cfg(not(feature = "loadable_extension"))]
if pkg_config::Config::new().probe(link_lib).is_err() {
// Otherwise just emit the bare minimum link commands.
println!("cargo:rustc-link-lib={}={link_lib}", find_link_mode());
@ -443,6 +470,7 @@ mod build_linked {
// request and hope that the library exists on the system paths. We used to
// output /usr/lib explicitly, but that can introduce other linking problems;
// see https://github.com/rusqlite/rusqlite/issues/207.
#[cfg(not(feature = "loadable_extension"))]
println!("cargo:rustc-link-lib={}={link_lib}", find_link_mode());
HeaderLocation::Wrapper
}
@ -470,7 +498,7 @@ mod bindings {
use std::path::Path;
static PREBUILT_BINDGENS: &[&str] = &["bindgen_3.14.0.rs"];
static PREBUILT_BINDGENS: &[&str] = &["bindgen_3.14.0"];
pub fn write_to_out_dir(_header: HeaderLocation, out_path: &Path) {
let name = PREBUILT_BINDGENS[PREBUILT_BINDGENS.len() - 1];
@ -522,7 +550,11 @@ mod bindings {
.disable_nested_struct_naming()
.trust_clang_mangling(false)
.header(header.clone())
.parse_callbacks(Box::new(SqliteTypeChooser))
.parse_callbacks(Box::new(SqliteTypeChooser));
if cfg!(feature = "loadable_extension") {
bindings = bindings.ignore_functions(); // see generate_functions
} else {
bindings = bindings
.blocklist_function("sqlite3_auto_extension")
.raw_line(
r#"extern "C" {
@ -551,6 +583,7 @@ mod bindings {
) -> ::std::os::raw::c_int;
}"#,
);
}
if cfg!(any(feature = "sqlcipher", feature = "bundled-sqlcipher")) {
bindings = bindings.clang_arg("-DSQLITE_HAS_CODEC");
@ -621,11 +654,204 @@ mod bindings {
.blocklist_item("__.*");
}
bindings
let bindings = bindings
.layout_tests(false)
.generate()
.unwrap_or_else(|_| panic!("could not run bindgen on header {}", header))
.write_to_file(out_path)
.unwrap_or_else(|_| panic!("could not run bindgen on header {}", header));
if cfg!(feature = "loadable_extension") {
let mut output = Vec::new();
bindings
.write(Box::new(&mut output))
.expect("could not write output of bindgen");
let mut output = String::from_utf8(output).expect("bindgen output was not UTF-8?!");
super::loadable_extension::generate_functions(&header, &mut output);
std::fs::write(out_path, output.as_bytes())
} else {
bindings.write_to_file(out_path)
}
.unwrap_or_else(|_| panic!("Could not write to {:?}", out_path));
}
}
#[cfg(all(feature = "buildtime_bindgen", feature = "loadable_extension"))]
mod loadable_extension {
use std::collections::HashMap;
/// try to generate similar rust code for all `#define sqlite3_xyz
/// sqlite3_api->abc` macros` in sqlite3ext.h
pub fn generate_functions(header: &str, output: &mut String) {
// (1) parse macros in sqlite3ext.h
let mappings = parse_macros(header);
// (2) parse sqlite3_api_routines fields from bindgen output
let ast: syn::File = syn::parse_str(output).expect("could not parse bindgen output");
let sqlite3_api_routines: syn::ItemStruct = ast
.items
.into_iter()
.find_map(|i| {
if let syn::Item::Struct(s) = i {
if s.ident == "sqlite3_api_routines" {
Some(s)
} else {
None
}
} else {
None
}
})
.expect("could not find sqlite3_api_routines");
let sqlite3_api_routines_ident = sqlite3_api_routines.ident;
let p_api = quote::format_ident!("p_api");
let mut stores = Vec::new();
// (3) `#define sqlite3_xyz sqlite3_api->abc` => `pub unsafe fn
// sqlite3_xyz(args) -> ty {...}` for each `abc` field:
for field in sqlite3_api_routines.fields {
let ident = field.ident.expect("unamed field");
let span = ident.span();
let name = ident.to_string();
if name == "vmprintf" || name == "xvsnprintf" || name == "str_vappendf" {
continue; // skip va_list
}
let sqlite3_name = mappings
.get(&name)
.unwrap_or_else(|| panic!("no mapping for {name}"));
let ptr_name =
syn::Ident::new(format!("__{}", sqlite3_name.to_uppercase()).as_ref(), span);
let sqlite3_fn_name = syn::Ident::new(sqlite3_name, span);
let method =
extract_method(&field.ty).unwrap_or_else(|| panic!("unexpected type for {name}"));
let arg_names: syn::punctuated::Punctuated<&syn::Ident, syn::token::Comma> = method
.inputs
.iter()
.map(|i| &i.name.as_ref().unwrap().0)
.collect();
let args = &method.inputs;
// vtab_config/sqlite3_vtab_config: ok
let varargs = &method.variadic;
if varargs.is_some() && "db_config" != name && "log" != name && "vtab_config" != name {
continue; // skip ...
}
let ty = &method.output;
let tokens = if "db_config" == name {
quote::quote! {
static #ptr_name: ::atomic::Atomic<Option<unsafe extern "C" fn(#args #varargs) #ty>> = ::atomic::Atomic::new(None);
pub unsafe fn #sqlite3_fn_name(#args arg3: ::std::os::raw::c_int, arg4: *mut ::std::os::raw::c_int) #ty {
let fun = #ptr_name.load(::atomic::Ordering::Acquire).expect("SQLite API not initialized");
(fun)(#arg_names, arg3, arg4)
}
}
} else if "log" == name {
quote::quote! {
static #ptr_name: ::atomic::Atomic<Option<unsafe extern "C" fn(#args #varargs) #ty>> = ::atomic::Atomic::new(None);
pub unsafe fn #sqlite3_fn_name(#args arg3: *const ::std::os::raw::c_char) #ty {
let fun = #ptr_name.load(::atomic::Ordering::Acquire).expect("SQLite API not initialized");
(fun)(#arg_names, arg3)
}
}
} else {
quote::quote! {
static #ptr_name: ::atomic::Atomic<Option<unsafe extern "C" fn(#args #varargs) #ty>> = ::atomic::Atomic::new(None);
pub unsafe fn #sqlite3_fn_name(#args) #ty {
let fun = #ptr_name.load(::atomic::Ordering::Acquire).expect("SQLite API not initialized or SQLite feature omitted");
(fun)(#arg_names)
}
}
};
output.push_str(&prettyplease::unparse(
&syn::parse2(tokens).expect("could not parse quote output"),
));
output.push('\n');
stores.push(quote::quote! {
#ptr_name.store(
(*#p_api).#ident,
::atomic::Ordering::Release,
);
});
}
// (4) generate rust code similar to SQLITE_EXTENSION_INIT2 macro
let tokens = quote::quote! {
/// Loadable extension initialization error
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum InitError {
/// Invalid sqlite3_api_routines pointer
NullApiPointer,
/// Version mismatch between the extension and the SQLite3 library
VersionMismatch{compile_time: i32, runtime: i32},
/// Invalid function pointer in one of sqlite3_api_routines fields
NullFunctionPointer,
}
impl ::std::fmt::Display for InitError {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
match *self {
InitError::NullApiPointer => write!(f, "Invalid sqlite3_api_routines pointer"),
InitError::VersionMismatch{compile_time, runtime} => write!(f, "SQLite version mismatch: {runtime} < {compile_time}"),
InitError::NullFunctionPointer => write!(f, "Some sqlite3_api_routines fields are null"),
}
}
}
/// Like SQLITE_EXTENSION_INIT2 macro
pub unsafe fn rusqlite_extension_init2(#p_api: *mut #sqlite3_api_routines_ident) -> ::std::result::Result<(),InitError> {
if #p_api.is_null() {
return Err(InitError::NullApiPointer);
}
if let Some(fun) = (*#p_api).libversion_number {
let version = fun();
if SQLITE_VERSION_NUMBER > version {
return Err(InitError::VersionMismatch{compile_time: SQLITE_VERSION_NUMBER, runtime: version});
}
} else {
return Err(InitError::NullFunctionPointer);
}
#(#stores)*
Ok(())
}
};
output.push_str(&prettyplease::unparse(
&syn::parse2(tokens).expect("could not parse quote output"),
));
output.push('\n');
}
// Load all `#define sqlite3_xyz sqlite3_api->abc` in sqlite3ext.h
// as a map `{abc => sqlite3_xyz}`
// See https://github.com/rust-lang/rust-bindgen/issues/2544
fn parse_macros(header: &str) -> HashMap<String, String> {
use regex::Regex;
use std::fs::File;
use std::io::{BufRead, BufReader};
let re = Regex::new(r"^#define\s+(sqlite3_\w+)\s+sqlite3_api->(\w+)").unwrap();
let f = File::open(header).expect("could not read sqlite3ext.h");
let f = BufReader::new(f);
let mut mappings = HashMap::new();
for line in f.lines() {
let line = line.expect("could not read line");
if let Some(caps) = re.captures(&line) {
mappings.insert(
caps.get(2).unwrap().as_str().to_owned(),
caps.get(1).unwrap().as_str().to_owned(),
);
}
}
mappings
}
fn extract_method(ty: &syn::Type) -> Option<&syn::TypeBareFn> {
match ty {
syn::Type::Path(tp) => tp.path.segments.last(),
_ => None,
}
.map(|seg| match &seg.arguments {
syn::PathArguments::AngleBracketed(args) => args.args.first(),
_ => None,
})?
.map(|arg| match arg {
syn::GenericArgument::Type(t) => Some(t),
_ => None,
})?
.map(|ty| match ty {
syn::Type::BareFn(r) => Some(r),
_ => None,
})?
}
}

File diff suppressed because it is too large Load Diff

View File

@ -3,10 +3,10 @@
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
echo "$SCRIPT_DIR"
cd "$SCRIPT_DIR" || { echo "fatal error" >&2; exit 1; }
cargo clean
mkdir -p "$SCRIPT_DIR/../target" "$SCRIPT_DIR/sqlite3"
cargo clean -p libsqlite3-sys
TARGET_DIR="$SCRIPT_DIR/../target"
export SQLITE3_LIB_DIR="$SCRIPT_DIR/sqlite3"
export SQLITE3_INCLUDE_DIR="$SQLITE3_LIB_DIR"
mkdir -p "$TARGET_DIR" "$SQLITE3_LIB_DIR"
# Download and extract amalgamation
SQLITE=sqlite-amalgamation-3420000
@ -16,13 +16,24 @@ unzip -p "$SQLITE.zip" "$SQLITE/sqlite3.h" > "$SQLITE3_LIB_DIR/sqlite3.h"
unzip -p "$SQLITE.zip" "$SQLITE/sqlite3ext.h" > "$SQLITE3_LIB_DIR/sqlite3ext.h"
rm -f "$SQLITE.zip"
# Regenerate bindgen file for sqlite3
export SQLITE3_INCLUDE_DIR="$SQLITE3_LIB_DIR"
# Regenerate bindgen file for sqlite3.h
rm -f "$SQLITE3_LIB_DIR/bindgen_bundled_version.rs"
cargo update
# Just to make sure there is only one bindgen.rs file in target dir
find "$SCRIPT_DIR/../target" -type f -name bindgen.rs -exec rm {} \;
find "$TARGET_DIR" -type f -name bindgen.rs -exec rm {} \;
env LIBSQLITE3_SYS_BUNDLING=1 cargo build --features "buildtime_bindgen session" --no-default-features
find "$SCRIPT_DIR/../target" -type f -name bindgen.rs -exec mv {} "$SQLITE3_LIB_DIR/bindgen_bundled_version.rs" \;
find "$TARGET_DIR" -type f -name bindgen.rs -exec mv {} "$SQLITE3_LIB_DIR/bindgen_bundled_version.rs" \;
# Regenerate bindgen file for sqlite3ext.h
# some sqlite3_api_routines fields are function pointers with va_list arg but currently stable Rust doesn't support this type.
# FIXME how to generate portable bindings without :
sed -i '' 's/va_list/void*/' "$SQLITE3_LIB_DIR/sqlite3ext.h"
rm -f "$SQLITE3_LIB_DIR/bindgen_bundled_version_ext.rs"
find "$TARGET_DIR" -type f -name bindgen.rs -exec rm {} \;
env LIBSQLITE3_SYS_BUNDLING=1 cargo build --features "buildtime_bindgen loadable_extension" --no-default-features
find "$TARGET_DIR" -type f -name bindgen.rs -exec mv {} "$SQLITE3_LIB_DIR/bindgen_bundled_version_ext.rs" \;
git checkout "$SQLITE3_LIB_DIR/sqlite3ext.h"
# Sanity checks
cd "$SCRIPT_DIR/.." || { echo "fatal error" >&2; exit 1; }

View File

@ -3,7 +3,7 @@
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
echo "$SCRIPT_DIR"
cd "$SCRIPT_DIR" || { echo "fatal error" >&2; exit 1; }
cargo clean
cargo clean -p libsqlite3-sys
mkdir -p "$SCRIPT_DIR/../target" "$SCRIPT_DIR/sqlcipher"
export SQLCIPHER_LIB_DIR="$SCRIPT_DIR/sqlcipher"
export SQLCIPHER_INCLUDE_DIR="$SQLCIPHER_LIB_DIR"

View File

@ -0,0 +1,2 @@
#include "sqlite3ext.h"

View File

@ -4,7 +4,7 @@ use std::os::raw::{c_char, c_int};
use std::path::Path;
use std::ptr;
use std::str;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Mutex};
use super::ffi;
@ -390,7 +390,7 @@ impl Drop for InnerConnection {
}
}
#[cfg(not(any(target_arch = "wasm32")))]
#[cfg(not(any(target_arch = "wasm32", feature = "loadable_extension")))]
static SQLITE_INIT: std::sync::Once = std::sync::Once::new();
pub static BYPASS_SQLITE_INIT: AtomicBool = AtomicBool::new(false);
@ -440,7 +440,9 @@ fn ensure_safe_sqlite_threading_mode() -> Result<()> {
Ok(())
}
} else {
#[cfg(not(feature = "loadable_extension"))]
SQLITE_INIT.call_once(|| {
use std::sync::atomic::Ordering;
if BYPASS_SQLITE_INIT.load(Ordering::Relaxed) {
return;
}

View File

@ -8,8 +8,7 @@ use std::ptr;
use std::time::Duration;
use super::ffi;
use crate::error::error_from_sqlite_code;
use crate::{Connection, Result};
use crate::Connection;
/// Set up the process-wide SQLite error logging callback.
///
@ -25,7 +24,8 @@ use crate::{Connection, Result};
/// * It must be threadsafe if SQLite is used in a multithreaded way.
///
/// cf [The Error And Warning Log](http://sqlite.org/errlog.html).
pub unsafe fn config_log(callback: Option<fn(c_int, &str)>) -> Result<()> {
#[cfg(not(feature = "loadable_extension"))]
pub unsafe fn config_log(callback: Option<fn(c_int, &str)>) -> crate::Result<()> {
extern "C" fn log_callback(p_arg: *mut c_void, err: c_int, msg: *const c_char) {
let c_slice = unsafe { CStr::from_ptr(msg).to_bytes() };
let callback: fn(c_int, &str) = unsafe { mem::transmute(p_arg) };
@ -48,7 +48,7 @@ pub unsafe fn config_log(callback: Option<fn(c_int, &str)>) -> Result<()> {
if rc == ffi::SQLITE_OK {
Ok(())
} else {
Err(error_from_sqlite_code(rc, None))
Err(crate::error::error_from_sqlite_code(rc, None))
}
}

View File

@ -1,11 +1,11 @@
//! Ensure we reject connections when SQLite is in single-threaded mode, as it
//! would violate safety if multiple Rust threads tried to use connections.
use rusqlite::ffi;
use rusqlite::Connection;
#[cfg(not(feature = "loadable_extension"))]
#[test]
fn test_error_when_singlethread_mode() {
use rusqlite::ffi;
use rusqlite::Connection;
// put SQLite into single-threaded mode
unsafe {
// Note: macOS system SQLite seems to return an error if you attempt to