Merge pull request #247 from jgallagher/no-buildtime-bindgen-by-default

Rework build process to not run bindgen by default

.travis.yml

@@ -37,4 +37,6 @@ script:
   - cargo test --features serde_json
   - cargo test --features bundled
   - cargo test --features "backup blob chrono functions limits load_extension serde_json trace"
+  - cargo test --features "backup blob chrono functions limits load_extension serde_json trace buildtime_bindgen"
   - cargo test --features "backup blob chrono functions limits load_extension serde_json trace bundled"
+  - cargo test --features "backup blob chrono functions limits load_extension serde_json trace bundled buildtime_bindgen"

Cargo.toml

@@ -19,11 +19,12 @@ name = "rusqlite"
 
 [features]
 load_extension = []
-backup = []
+backup = ["libsqlite3-sys/min_sqlite_version_3_6_11"]
-blob = []
+blob = ["libsqlite3-sys/min_sqlite_version_3_7_4"]
-functions = []
+functions = ["libsqlite3-sys/min_sqlite_version_3_7_3"]
-trace = []
+trace = ["libsqlite3-sys/min_sqlite_version_3_6_23"]
 bundled = ["libsqlite3-sys/bundled"]
+buildtime_bindgen = ["libsqlite3-sys/buildtime_bindgen"]
 limits = []
 
 [dependencies]

Changelog.md

@@ -1,6 +1,9 @@
-# Version 0.10.1 (UPCOMING)
+# Version 0.10.1 (2017-03-03)
 
 * Updates the `bundled` SQLite version to 3.17.0.
+* Changes the build process to no longer require `bindgen`. This should improve
+  build times and no longer require a new-ish Clang. See the README for more
+  details.
 
 # Version 0.10.0 (2017-02-28)
 

README.md

@@ -91,6 +91,53 @@ features](http://doc.crates.io/manifest.html#the-features-section). They are:
   `Value` type from the [`serde_json` crate](https://crates.io/crates/serde_json).
 * `bundled` uses a bundled version of sqlite3.  This is a good option for cases where linking to sqlite3 is complicated, such as Windows.
 
+## Notes on building rusqlite and libsqlite3-sys
+
+`libsqlite3-sys` is a separate crate from `rusqlite` that provides the Rust
+declarations for SQLite's C API. By default, `libsqlite3-sys` attempts to use
+pkg-config to find a SQLite library that already exists on your system. You can
+adjust this behavior in a couple of ways:
+
+* If you use the `bundled` feature, `libsqlite3-sys` will use the
+  [gcc](https://crates.io/crates/gcc) crate to compile SQLite from source and
+  link against that. This source is embedded in the `libsqlite3-sys` crate and
+  is currently SQLite 3.17.0 (as of `rusqlite` 0.10.1 / `libsqlite3-sys`
+  0.7.1).  This is probably the simplest solution to any build problems.
+* You can set the `SQLITE3_LIB_DIR` to point to directory containing the SQLite
+  library.
+
+### Binding generation
+
+We use [bindgen](https://crates.io/crates/bindgen) to generate the Rust
+declarations from SQLite's C header file. `bindgen`
+[recommends](https://github.com/servo/rust-bindgen#library-usage-with-buildrs)
+running this as part of the build process of libraries that used this. We tried
+this briefly (`rusqlite` 0.10.0, specifically), but it had some annoyances:
+
+* The build time for `libsqlite3-sys` (and therefore `rusqlite`) increased
+  dramatically.
+* Running `bindgen` requires a relatively-recent version of Clang, which many
+  systems do not have installed by default.
+* Running `bindgen` also requires the SQLite header file to be present.
+
+As of `rusqlite` 0.10.1, we avoid running `bindgen` at build-time by shipping
+pregenerated bindings for several versions of SQLite. When compiling
+`rusqlite`, we use your selected Cargo features to pick the bindings for the
+minimum SQLite version that supports your chosen features. If you are using
+`libsqlite3-sys` directly, you can use the same features to choose which
+pregenerated bindings are chosen:
+
+* `min_sqlite_version_3_6_8` - SQLite 3.6.8 bindings (this is the default)
+* `min_sqlite_version_3_6_11` - SQLite 3.6.11 bindings
+* `min_sqlite_version_3_6_23` - SQLite 3.6.23 bindings
+* `min_sqlite_version_3_7_3` - SQLite 3.7.3 bindings
+* `min_sqlite_version_3_7_4` - SQLite 3.7.4 bindings
+
+If you use the `bundled` feature, you will get pregenerated bindings for the
+bundled version of SQLite. If you need other specific pregenerated binding
+versions, please file an issue. If you want to run `bindgen` at buildtime to
+produce your own bindings, use the `buildtime_bindgen` Cargo feature.
+
 ## Author
 
 John Gallagher, johnkgallagher@gmail.com

appveyor.yml

@@ -21,7 +21,9 @@ test_script:
   - cargo test --lib --verbose
   - cargo test --lib --verbose --features bundled
   - cargo test --lib --features "backup blob chrono functions limits load_extension serde_json trace"
+  - cargo test --lib --features "backup blob chrono functions limits load_extension serde_json trace buildtime_bindgen"
   - cargo test --lib --features "backup blob chrono functions limits load_extension serde_json trace bundled"
+  - cargo test --lib --features "backup blob chrono functions limits load_extension serde_json trace bundled buildtime_bindgen"
 
 cache:
   - C:\Users\appveyor\.cargo

libsqlite3-sys/Cargo.toml

@@ -11,9 +11,16 @@ keywords = ["sqlite", "database", "ffi"]
 categories = ["database", "external-ffi-bindings"]
 
 [features]
-bundled = []
+default = ["min_sqlite_version_3_6_8"]
+bundled = ["gcc"]
+buildtime_bindgen = ["bindgen", "pkg-config"]
+min_sqlite_version_3_6_8 = ["pkg-config"]
+min_sqlite_version_3_6_11 = ["pkg-config"]
+min_sqlite_version_3_6_23 = ["pkg-config"]
+min_sqlite_version_3_7_3 = ["pkg-config"]
+min_sqlite_version_3_7_4 = ["pkg-config"]
 
 [build-dependencies]
-bindgen = "0.21"
+bindgen = { version = "0.21", optional = true }
-pkg-config = "0.3"
+pkg-config = { version = "0.3", optional = true }
-gcc = "0.3"
+gcc = { version = "0.3", optional = true }

libsqlite3-sys/build.rs

@@ -1,116 +1,193 @@
-extern crate bindgen;
-extern crate gcc;
-extern crate pkg_config;
-
-use std::env;
-use std::io::Write;
-use std::fs::OpenOptions;
-use bindgen::chooser::{TypeChooser, IntKind};
-use std::path::Path;
-
-#[derive(Debug)]
-struct SqliteTypeChooser;
-
-impl TypeChooser for SqliteTypeChooser {
-    fn int_macro(&self, _name: &str, value: i64) -> Option<IntKind> {
-        if value >= i32::min_value() as i64 && value <= i32::max_value() as i64 {
-            Some(IntKind::I32)
-        } else {
-            None
-        }
-    }
-}
-
-fn run_bindgen<T: Into<String>>(header: T) {
-    let out_dir = env::var("OUT_DIR").unwrap();
-    let header = header.into();
-    let mut output = Vec::new();
-    bindgen::builder()
-        .header(header.clone())
-        .type_chooser(Box::new(SqliteTypeChooser))
-        .generate()
-        .expect(&format!("could not run bindgen on header {}", header))
-        .write(Box::new(&mut output))
-        .expect("could not write output of bindgen");
-    let mut output = String::from_utf8(output).expect("bindgen output was not UTF-8?!");
-
-    // rusqlite's functions feature ors in the SQLITE_DETERMINISTIC flag when it can. This flag
-    // was added in SQLite 3.8.3, but oring it in in prior versions of SQLite is harmless. We
-    // don't want to not build just because this flag is missing (e.g., if we're linking against
-    // SQLite 3.7.x), so append the flag manually if it isn't present in bindgen's output.
-    if !output.contains("pub const SQLITE_DETERMINISTIC:") {
-        output.push_str("\npub const SQLITE_DETERMINISTIC: i32 = 2048;\n");
-    }
-
-    let path = Path::new(&out_dir).join("bindgen.rs");
-
-    let mut file = OpenOptions::new()
-        .write(true)
-        .truncate(true)
-        .create(true)
-        .open(path.clone())
-        .expect(&format!("Could not write to {:?}", path));
-
-    file.write_all(output.as_bytes()).expect(&format!("Could not write to {:?}", path));
-}
-
-#[cfg(not(feature = "bundled"))]
 fn main() {
-    // Allow users to specify where to find SQLite.
+    build::main();
-    if let Ok(dir) = env::var("SQLITE3_LIB_DIR") {
-        let mut header = env::var("SQLITE3_INCLUDE_DIR")
-            .expect("SQLITE3_INCLUDE_DIR must be set if SQLITE3_LIB_DIR is set");
-        header.push_str("/sqlite3.h");
-        run_bindgen(header);
-        println!("cargo:rustc-link-lib=sqlite3");
-        println!("cargo:rustc-link-search={}", dir);
-        return;
-    }
-
-    // See if pkg-config can do everything for us.
-    match pkg_config::Config::new().print_system_libs(false).probe("sqlite3") {
-        Ok(mut lib) => {
-            if let Some(mut header) = lib.include_paths.pop() {
-                header.push("sqlite3.h");
-                run_bindgen(header.to_string_lossy());
-            } else {
-                run_bindgen("wrapper.h");
-            }
-        }
-        Err(_) => {
-            // No env var set and pkg-config couldn't help; just output the link-lib
-            // request and hope that the library exists on the system paths. We used to
-            // output /usr/lib explicitly, but that can introduce other linking problems; see
-            // https://github.com/jgallagher/rusqlite/issues/207.
-            println!("cargo:rustc-link-lib=sqlite3");
-            run_bindgen("wrapper.h");
-        }
-    }
 }
 
 #[cfg(feature = "bundled")]
-fn main() {
+mod build {
-    run_bindgen("sqlite3/sqlite3.h");
+    extern crate gcc;
+    use std::{env, fs};
+    use std::path::Path;
 
-    gcc::Config::new()
+    pub fn main() {
-        .file("sqlite3/sqlite3.c")
+        let out_dir = env::var("OUT_DIR").unwrap();
-        .flag("-DSQLITE_CORE")
+        let out_path = Path::new(&out_dir).join("bindgen.rs");
-        .flag("-DSQLITE_DEFAULT_FOREIGN_KEYS=1")
+        fs::copy("sqlite3/bindgen_bundled_version.rs", out_path)
-        .flag("-DSQLITE_ENABLE_API_ARMOR")
+            .expect("Could not copy bindings to output directory");
-        .flag("-DSQLITE_ENABLE_COLUMN_METADATA")
+
-        .flag("-DSQLITE_ENABLE_DBSTAT_VTAB")
+        gcc::Config::new()
-        .flag("-DSQLITE_ENABLE_FTS3")
+            .file("sqlite3/sqlite3.c")
-        .flag("-DSQLITE_ENABLE_FTS3_PARENTHESIS")
+            .flag("-DSQLITE_CORE")
-        .flag("-DSQLITE_ENABLE_FTS5")
+            .flag("-DSQLITE_DEFAULT_FOREIGN_KEYS=1")
-        .flag("-DSQLITE_ENABLE_JSON1")
+            .flag("-DSQLITE_ENABLE_API_ARMOR")
-        .flag("-DSQLITE_ENABLE_LOAD_EXTENSION=1")
+            .flag("-DSQLITE_ENABLE_COLUMN_METADATA")
-        .flag("-DSQLITE_ENABLE_MEMORY_MANAGEMENT")
+            .flag("-DSQLITE_ENABLE_DBSTAT_VTAB")
-        .flag("-DSQLITE_ENABLE_RTREE")
+            .flag("-DSQLITE_ENABLE_FTS3")
-        .flag("-DSQLITE_ENABLE_STAT2")
+            .flag("-DSQLITE_ENABLE_FTS3_PARENTHESIS")
-        .flag("-DSQLITE_ENABLE_STAT4")
+            .flag("-DSQLITE_ENABLE_FTS5")
-        .flag("-DSQLITE_HAVE_ISNAN")
+            .flag("-DSQLITE_ENABLE_JSON1")
-        .flag("-DSQLITE_SOUNDEX")
+            .flag("-DSQLITE_ENABLE_LOAD_EXTENSION=1")
-        .flag("-DSQLITE_THREADSAFE=1")
+            .flag("-DSQLITE_ENABLE_MEMORY_MANAGEMENT")
-        .flag("-DSQLITE_USE_URI")
+            .flag("-DSQLITE_ENABLE_RTREE")
-        .compile("libsqlite3.a");
+            .flag("-DSQLITE_ENABLE_STAT2")
+            .flag("-DSQLITE_ENABLE_STAT4")
+            .flag("-DSQLITE_HAVE_ISNAN")
+            .flag("-DSQLITE_SOUNDEX")
+            .flag("-DSQLITE_THREADSAFE=1")
+            .flag("-DSQLITE_USE_URI")
+            .compile("libsqlite3.a");
+    }
+}
+
+#[cfg(not(feature = "bundled"))]
+mod build {
+    extern crate pkg_config;
+
+    use std::env;
+
+    pub enum HeaderLocation {
+        FromEnvironment,
+        Wrapper,
+        FromPath(String),
+    }
+
+    impl From<HeaderLocation> for String {
+        fn from(header: HeaderLocation) -> String {
+            match header {
+                HeaderLocation::FromEnvironment => {
+                    let mut header = env::var("SQLITE3_INCLUDE_DIR")
+                        .expect("SQLITE3_INCLUDE_DIR must be set if SQLITE3_LIB_DIR is set");
+                    header.push_str("/sqlite3.h");
+                    header
+                }
+                HeaderLocation::Wrapper => "wrapper.h".into(),
+                HeaderLocation::FromPath(path) => path,
+            }
+        }
+    }
+
+    pub fn main() {
+        let header = find_sqlite();
+        bindings::write_to_out_dir(header);
+    }
+
+    // Prints the necessary cargo link commands and returns the path to the header.
+    fn find_sqlite() -> HeaderLocation {
+        // Allow users to specify where to find SQLite.
+        if let Ok(dir) = env::var("SQLITE3_LIB_DIR") {
+            println!("cargo:rustc-link-lib=sqlite3");
+            println!("cargo:rustc-link-search={}", dir);
+            return HeaderLocation::FromEnvironment;
+        }
+
+        // See if pkg-config can do everything for us.
+        match pkg_config::Config::new().print_system_libs(false).probe("sqlite3") {
+            Ok(mut lib) => {
+                if let Some(mut header) = lib.include_paths.pop() {
+                    header.push("sqlite3.h");
+                    HeaderLocation::FromPath(header.to_string_lossy().into())
+                } else {
+                    HeaderLocation::Wrapper
+                }
+            }
+            Err(_) => {
+                // No env var set and pkg-config couldn't help; just output the link-lib
+                // request and hope that the library exists on the system paths. We used to
+                // output /usr/lib explicitly, but that can introduce other linking problems; see
+                // https://github.com/jgallagher/rusqlite/issues/207.
+                println!("cargo:rustc-link-lib=sqlite3");
+                HeaderLocation::Wrapper
+            }
+        }
+    }
+
+    #[cfg(not(feature = "buildtime_bindgen"))]
+    mod bindings {
+        use super::HeaderLocation;
+
+        use std::{env, fs};
+        use std::path::Path;
+
+        #[cfg_attr(rustfmt, rustfmt_skip)]
+        static PREBUILT_BINDGEN_PATHS: &'static [&'static str] = &[
+            "bindgen-bindings/bindgen_3.6.8.rs",
+
+            #[cfg(feature = "min_sqlite_version_3_6_11")]
+            "bindgen-bindings/bindgen_3.6.11.rs",
+
+            #[cfg(feature = "min_sqlite_version_3_6_23")]
+            "bindgen-bindings/bindgen_3.6.23.rs",
+
+            #[cfg(feature = "min_sqlite_version_3_7_3")]
+            "bindgen-bindings/bindgen_3.7.3.rs",
+
+            #[cfg(feature = "min_sqlite_version_3_7_4")]
+            "bindgen-bindings/bindgen_3.7.4.rs",
+        ];
+
+        pub fn write_to_out_dir(_header: HeaderLocation) {
+            let out_dir = env::var("OUT_DIR").unwrap();
+            let out_path = Path::new(&out_dir).join("bindgen.rs");
+            let in_path = PREBUILT_BINDGEN_PATHS[PREBUILT_BINDGEN_PATHS.len() - 1];
+            fs::copy(in_path, out_path).expect("Could not copy bindings to output directory");
+        }
+    }
+
+    #[cfg(feature = "buildtime_bindgen")]
+    mod bindings {
+        extern crate bindgen;
+
+        use self::bindgen::chooser::{TypeChooser, IntKind};
+        use super::HeaderLocation;
+
+        use std::env;
+        use std::io::Write;
+        use std::fs::OpenOptions;
+        use std::path::Path;
+
+        #[derive(Debug)]
+        struct SqliteTypeChooser;
+
+        impl TypeChooser for SqliteTypeChooser {
+            fn int_macro(&self, _name: &str, value: i64) -> Option<IntKind> {
+                if value >= i32::min_value() as i64 && value <= i32::max_value() as i64 {
+                    Some(IntKind::I32)
+                } else {
+                    None
+                }
+            }
+        }
+
+        pub fn write_to_out_dir(header: HeaderLocation) {
+            let header: String = header.into();
+            let out_dir = env::var("OUT_DIR").unwrap();
+            let mut output = Vec::new();
+            bindgen::builder()
+                .header(header.clone())
+                .type_chooser(Box::new(SqliteTypeChooser))
+                .generate()
+                .expect(&format!("could not run bindgen on header {}", header))
+                .write(Box::new(&mut output))
+                .expect("could not write output of bindgen");
+            let mut output = String::from_utf8(output).expect("bindgen output was not UTF-8?!");
+
+            // rusqlite's functions feature ors in the SQLITE_DETERMINISTIC flag when it can. This flag
+            // was added in SQLite 3.8.3, but oring it in in prior versions of SQLite is harmless. We
+            // don't want to not build just because this flag is missing (e.g., if we're linking against
+            // SQLite 3.7.x), so append the flag manually if it isn't present in bindgen's output.
+            if !output.contains("pub const SQLITE_DETERMINISTIC:") {
+                output.push_str("\npub const SQLITE_DETERMINISTIC: i32 = 2048;\n");
+            }
+
+            let path = Path::new(&out_dir).join("bindgen.rs");
+
+            let mut file = OpenOptions::new()
+                .write(true)
+                .truncate(true)
+                .create(true)
+                .open(path.clone())
+                .expect(&format!("Could not write to {:?}", path));
+
+            file.write_all(output.as_bytes()).expect(&format!("Could not write to {:?}", path));
+        }
+    }
 }