matrix_sdk_sqlite/
lib.rs

1// Copyright 2022 The Matrix.org Foundation C.I.C.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![cfg_attr(
16    not(any(
17        feature = "state-store",
18        feature = "crypto-store",
19        feature = "event-cache-store",
20        feature = "media-store"
21    )),
22    allow(dead_code, unused_imports)
23)]
24
25mod connection;
26#[cfg(feature = "crypto-store")]
27mod crypto_store;
28mod error;
29#[cfg(feature = "event-cache-store")]
30mod event_cache_store;
31#[cfg(feature = "media-store")]
32mod media_store;
33#[cfg(feature = "state-store")]
34mod state_store;
35mod utils;
36use std::{
37    cmp::max,
38    fmt,
39    path::{Path, PathBuf},
40};
41
42use deadpool::managed::PoolConfig;
43use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
44
45#[cfg(feature = "crypto-store")]
46pub use self::crypto_store::SqliteCryptoStore;
47pub use self::error::OpenStoreError;
48#[cfg(feature = "event-cache-store")]
49pub use self::event_cache_store::{
50    DATABASE_NAME as EVENT_CACHE_STORE_DATABASE_NAME, SqliteEventCacheStore,
51};
52#[cfg(feature = "media-store")]
53pub use self::media_store::{DATABASE_NAME as MEDIA_STORE_DATABASE_NAME, SqliteMediaStore};
54#[cfg(feature = "state-store")]
55pub use self::state_store::{DATABASE_NAME as STATE_STORE_DATABASE_NAME, SqliteStateStore};
56
57#[cfg(feature = "uniffi")]
58uniffi::setup_scaffolding!();
59
60#[cfg(test)]
61matrix_sdk_test_utils::init_tracing_for_tests!();
62
63/// An enum used to store the secret that gives access to a store
64#[derive(Clone, Debug, PartialEq, Zeroize, ZeroizeOnDrop)]
65pub enum Secret {
66    // Cryptographic key used to open the store
67    Key(Zeroizing<Vec<u8>>),
68    // Passphrase used to open the store, ideally human chosen
69    PassPhrase(Zeroizing<String>),
70    // Randomly generated passphrase, for which the store caches a
71    // cheaply-derivable copy of its cipher and skips derivation on later opens
72    HighEntropyPassPhrase {
73        key: Zeroizing<Vec<u8>>,
74        #[zeroize(skip)]
75        base64_variant: Base64Variant,
76    },
77}
78
79/// Enum controlling how the high-entropy passphrase used to be created on the
80/// client side.
81///
82/// This allows us to replicate how a random key was converted into a passphrase
83/// to migrate from said passphrase to the plain key.
84#[derive(Clone, Copy, Debug, PartialEq, Eq)]
85#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))]
86pub enum Base64Variant {
87    /// Unpadded base64 was used to create the high-entropy passphrase.
88    Unpadded,
89    /// Standard padded base64 was used to create the high-entropy passphrase.
90    Padded,
91}
92
93/// A configuration structure used for opening a store.
94#[derive(Clone)]
95pub struct SqliteStoreConfig {
96    /// Path to the database, without the file name.
97    path: PathBuf,
98    /// Secret to open the store, if any
99    secret: Option<Secret>,
100    /// The pool configuration for [`deadpool`].
101    pool_config: PoolConfig,
102    /// The runtime configuration to apply when opening an SQLite connection.
103    runtime_config: RuntimeConfig,
104}
105
106impl fmt::Debug for SqliteStoreConfig {
107    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
108        formatter
109            .debug_struct("SqliteStoreConfig")
110            .field("path", &self.path)
111            .field("pool_config", &self.pool_config)
112            .field("runtime_config", &self.runtime_config)
113            .finish_non_exhaustive()
114    }
115}
116
117/// The minimum size of the connections pool.
118///
119/// We need at least 2 connections: one connection for write operations, and one
120/// connection for read operations.
121const POOL_MINIMUM_SIZE: usize = 2;
122
123impl SqliteStoreConfig {
124    /// Create a new [`SqliteStoreConfig`] with a path representing the
125    /// directory containing the store database.
126    pub fn new<P>(path: P) -> Self
127    where
128        P: AsRef<Path>,
129    {
130        Self {
131            path: path.as_ref().to_path_buf(),
132            pool_config: PoolConfig::new(max(POOL_MINIMUM_SIZE, num_cpus::get_physical() * 4)),
133            runtime_config: RuntimeConfig::default(),
134            secret: None,
135        }
136    }
137
138    /// Similar to [`SqliteStoreConfig::new`], but with defaults tailored for a
139    /// low memory usage environment.
140    ///
141    /// The following defaults are set:
142    ///
143    /// - The `pool_max_size` is set to the number of physical CPU, so one
144    ///   connection per physical thread,
145    /// - The `cache_size` is set to 500Kib,
146    /// - The `journal_size_limit` is set to 2Mib.
147    pub fn with_low_memory_config<P>(path: P) -> Self
148    where
149        P: AsRef<Path>,
150    {
151        Self::new(path)
152            // Maximum one connection per physical thread.
153            .pool_max_size(num_cpus::get_physical())
154            // Cache size is 500Kib.
155            .cache_size(500_000)
156            // Journal size limit is 2Mib.
157            .journal_size_limit(2_000_000)
158    }
159
160    /// Override the path.
161    pub fn path<P>(mut self, path: P) -> Self
162    where
163        P: AsRef<Path>,
164    {
165        self.path = path.as_ref().to_path_buf();
166        self
167    }
168
169    /// Define the passphrase if the store is encoded.
170    ///
171    /// Assumed to be possibly human-chosen, so an expensive derivation is run
172    /// over it on every open. If it is randomly generated, use
173    /// [`SqliteStoreConfig::high_entropy_passphrase`] instead.
174    pub fn passphrase(mut self, passphrase: Option<&str>) -> Self {
175        self.secret =
176            passphrase.map(|passphrase| Secret::PassPhrase(Zeroizing::new(passphrase.to_owned())));
177        self
178    }
179
180    /// Define the passphrase if the store is encoded, declaring that it was
181    /// randomly generated rather than chosen by a human.
182    ///
183    /// Do NOT use this with human-chosen passphrases, as doing so would remove
184    /// their brute-force protection.
185    ///
186    /// This migrates a passphrase-based store whose passphrase was created by
187    /// base64-encoding a randomly generated key to a key-based setup.
188    ///
189    /// Once this function has been called, [`SqliteStoreConfig::passphrase`]
190    /// can no longer be used with the passphrase.
191    ///
192    /// [`SqliteStoreConfig::key`] can be used with the original key, before it
193    /// was base64-encoded.
194    pub fn high_entropy_passphrase(
195        mut self,
196        passphrase: Option<&[u8]>,
197        base64_variant: Base64Variant,
198    ) -> Self {
199        if let Some(passphrase) = passphrase {
200            let key = Zeroizing::new(passphrase.to_vec());
201            self.secret = Some(Secret::HighEntropyPassPhrase { key, base64_variant });
202        }
203
204        self
205    }
206
207    /// Define the key if the store is encoded.
208    ///
209    /// Assumed to be high entropy so no derivation is run over it.
210    pub fn key(mut self, key: Option<&[u8]>) -> Self {
211        if let Some(key) = key {
212            let key = Zeroizing::new(key.to_vec());
213            self.secret = Some(Secret::Key(key));
214        }
215
216        self
217    }
218
219    /// Define the maximum pool size for [`deadpool`].
220    ///
221    /// See [`deadpool::managed::PoolConfig::max_size`] to learn more.
222    pub fn pool_max_size(mut self, max_size: usize) -> Self {
223        self.pool_config.max_size = max(POOL_MINIMUM_SIZE, max_size);
224        self
225    }
226
227    /// Optimize the database.
228    ///
229    /// The SQLite documentation recommends to run this regularly and after any
230    /// schema change. The easiest is to do it consistently when the store is
231    /// constructed, after eventual migrations.
232    ///
233    /// See [`PRAGMA optimize`] to learn more.
234    ///
235    /// The default value is `true`.
236    ///
237    /// [`PRAGMA optimize`]: https://www.sqlite.org/pragma.html#pragma_optimize
238    pub fn optimize(mut self, optimize: bool) -> Self {
239        self.runtime_config.optimize = optimize;
240        self
241    }
242
243    /// Define the maximum size in **bytes** the SQLite cache can use.
244    ///
245    /// See [`PRAGMA cache_size`] to learn more.
246    ///
247    /// The default value is 2Mib.
248    ///
249    /// [`PRAGMA cache_size`]: https://www.sqlite.org/pragma.html#pragma_cache_size
250    pub fn cache_size(mut self, cache_size: u32) -> Self {
251        self.runtime_config.cache_size = cache_size;
252        self
253    }
254
255    /// Limit the size of the WAL file, in **bytes**.
256    ///
257    /// By default, while the DB connections of the databases are open,
258    /// [the size of the WAL file can keep increasing][size_wal_file] depending
259    /// on the size needed for the transactions. A critical case is `VACUUM`
260    /// which basically writes the content of the DB file to the WAL file before
261    /// writing it back to the DB file, so we end up taking twice the size of
262    /// the database.
263    ///
264    /// By setting this limit, the WAL file is truncated after its content is
265    /// written to the database, if it is bigger than the limit.
266    ///
267    /// See [`PRAGMA journal_size_limit`] to learn more. The value `limit`
268    /// corresponds to `N` in `PRAGMA journal_size_limit = N`.
269    ///
270    /// The default value is 10Mib.
271    ///
272    /// [size_wal_file]: https://www.sqlite.org/wal.html#avoiding_excessively_large_wal_files
273    /// [`PRAGMA journal_size_limit`]: https://www.sqlite.org/pragma.html#pragma_journal_size_limit
274    pub fn journal_size_limit(mut self, limit: u32) -> Self {
275        self.runtime_config.journal_size_limit = limit;
276        self
277    }
278
279    /// Define how often SQLite syncs the database to the storage device.
280    ///
281    /// [`Synchronous::Normal`] is faster, but a power loss can drop the most
282    /// recent transactions. Defaults to [`Synchronous::Full`].
283    ///
284    /// See [`PRAGMA synchronous`] to learn more.
285    ///
286    /// [`PRAGMA synchronous`]: https://www.sqlite.org/pragma.html#pragma_synchronous
287    pub fn synchronous(mut self, synchronous: Synchronous) -> Self {
288        self.runtime_config.synchronous = synchronous;
289        self
290    }
291
292    /// Returns the pool configuration.
293    pub(crate) fn pool_config(&self) -> PoolConfig {
294        self.pool_config
295    }
296
297    /// Returns the runtime configuration.
298    pub(crate) fn runtime_config(&self) -> RuntimeConfig {
299        self.runtime_config
300    }
301
302    /// Build a pool of active connections to a particular database.
303    pub fn build_pool_of_connections(
304        &self,
305        database_name: &str,
306    ) -> Result<connection::Pool, connection::CreatePoolError> {
307        let path = self.path.join(database_name);
308        let manager = connection::Manager::new(path, self.runtime_config);
309
310        connection::Pool::builder(manager)
311            .config(self.pool_config)
312            .runtime(connection::RUNTIME)
313            .build()
314            .map_err(connection::CreatePoolError::Build)
315    }
316}
317
318/// How often SQLite syncs the database to the storage device.
319///
320/// See [`PRAGMA synchronous`] to learn more.
321///
322/// [`PRAGMA synchronous`]: https://www.sqlite.org/pragma.html#pragma_synchronous
323#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
324pub enum Synchronous {
325    /// SQLite never syncs. A crash of the operating system can corrupt the
326    /// database.
327    Off,
328
329    /// SQLite syncs at the most critical moments. In WAL mode, that is one
330    /// `fsync` per checkpoint.
331    Normal,
332
333    /// SQLite syncs at every transaction. In WAL mode, that is one `fsync` of
334    /// the WAL file per commit. This is the default of SQLite.
335    #[default]
336    Full,
337
338    /// Like [`Synchronous::Full`], plus one `fsync` of the directory when a
339    /// rollback journal is deleted. This has no effect in WAL mode.
340    Extra,
341}
342
343impl Synchronous {
344    fn as_str(self) -> &'static str {
345        match self {
346            Self::Off => "OFF",
347            Self::Normal => "NORMAL",
348            Self::Full => "FULL",
349            Self::Extra => "EXTRA",
350        }
351    }
352}
353
354/// This type represents values to set at runtime when a database is opened.
355///
356/// The per-connection part is applied by
357/// [`connection::Manager`] to every connection it creates. The rest is applied
358/// by [`utils::SqliteAsyncConnExt::apply_runtime_config`].
359#[derive(Clone, Copy, Debug)]
360pub(crate) struct RuntimeConfig {
361    /// If `true`, [`utils::SqliteAsyncConnExt::optimize`] will be called.
362    optimize: bool,
363
364    /// Regardless of the value, `PRAGMA cache_size` will always be set to this
365    /// value.
366    cache_size: u32,
367
368    /// Regardless of the value, `PRAGMA journal_size_limit` will always be set
369    /// to this value.
370    journal_size_limit: u32,
371
372    /// Regardless of the value, `PRAGMA synchronous` will always be set to this
373    /// value.
374    synchronous: Synchronous,
375}
376
377impl RuntimeConfig {
378    /// The pragmas to run on every new connection, as `synchronous`,
379    /// `cache_size` and `journal_size_limit` all apply to one connection only.
380    pub(crate) fn connection_pragmas(&self) -> String {
381        // `N` in `PRAGMA cache_size = -N` is expressed in kibibytes, while
382        // `cache_size` is expressed in bytes.
383        let cache_size_in_kib = self.cache_size / 1024;
384
385        format!(
386            "PRAGMA synchronous = {}; \
387             PRAGMA cache_size = -{cache_size_in_kib}; \
388             PRAGMA journal_size_limit = {};",
389            self.synchronous.as_str(),
390            self.journal_size_limit,
391        )
392    }
393}
394
395impl Default for RuntimeConfig {
396    fn default() -> Self {
397        Self {
398            // Optimize is always applied.
399            optimize: true,
400            // A cache of 2Mib.
401            cache_size: 2_000_000,
402            // A limit of 10Mib.
403            journal_size_limit: 10_000_000,
404            synchronous: Synchronous::default(),
405        }
406    }
407}
408
409#[cfg(test)]
410mod tests {
411    use std::{
412        ops::Not,
413        path::{Path, PathBuf},
414    };
415
416    use zeroize::Zeroizing;
417
418    use super::{POOL_MINIMUM_SIZE, Secret, SqliteStoreConfig, Synchronous};
419
420    #[test]
421    fn test_new() {
422        let store_config = SqliteStoreConfig::new(Path::new("foo"));
423
424        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical() * 4);
425        assert!(store_config.runtime_config.optimize);
426        assert_eq!(store_config.runtime_config.cache_size, 2_000_000);
427        assert_eq!(store_config.runtime_config.journal_size_limit, 10_000_000);
428        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Full);
429    }
430
431    #[test]
432    fn test_with_low_memory_config() {
433        let store_config = SqliteStoreConfig::with_low_memory_config(Path::new("foo"));
434
435        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical());
436        assert!(store_config.runtime_config.optimize);
437        assert_eq!(store_config.runtime_config.cache_size, 500_000);
438        assert_eq!(store_config.runtime_config.journal_size_limit, 2_000_000);
439        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Full);
440    }
441
442    #[test]
443    fn test_store_config_when_passphrase() {
444        let store_config = SqliteStoreConfig::new(Path::new("foo"))
445            .passphrase(Some("bar"))
446            .pool_max_size(42)
447            .optimize(false)
448            .cache_size(43)
449            .journal_size_limit(44)
450            .synchronous(Synchronous::Off);
451
452        assert_eq!(store_config.path, PathBuf::from("foo"));
453        assert_eq!(store_config.secret, Some(Secret::PassPhrase("bar".to_owned().into())));
454        assert_eq!(store_config.pool_config.max_size, 42);
455        assert!(store_config.runtime_config.optimize.not());
456        assert_eq!(store_config.runtime_config.cache_size, 43);
457        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
458        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Off);
459    }
460
461    #[test]
462    fn test_store_config_when_key() {
463        let store_config = SqliteStoreConfig::new(Path::new("foo"))
464            .key(Some(&[
465                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
466                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
467            ]))
468            .pool_max_size(42)
469            .optimize(false)
470            .cache_size(43)
471            .journal_size_limit(44)
472            .synchronous(Synchronous::Extra);
473
474        assert_eq!(store_config.path, PathBuf::from("foo"));
475        assert_eq!(
476            store_config.secret,
477            Some(Secret::Key(Zeroizing::new(vec![
478                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
479                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
480            ])))
481        );
482        assert_eq!(store_config.pool_config.max_size, 42);
483        assert!(store_config.runtime_config.optimize.not());
484        assert_eq!(store_config.runtime_config.cache_size, 43);
485        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
486        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Extra);
487    }
488
489    #[test]
490    fn test_store_config_path() {
491        let store_config = SqliteStoreConfig::new(Path::new("foo")).path(Path::new("bar"));
492
493        assert_eq!(store_config.path, PathBuf::from("bar"));
494    }
495
496    #[test]
497    fn test_pool_size_has_a_minimum() {
498        let store_config = SqliteStoreConfig::new(Path::new("foo")).pool_max_size(1);
499
500        assert_eq!(store_config.pool_config.max_size, POOL_MINIMUM_SIZE);
501    }
502}
matrix_sdk_sqlite/lib.rs

matrix_sdk_sqlite/
lib.rs
