matrix_sdk_sqlite/
lib.rs

1// Copyright 2022 The Matrix.org Foundation C.I.C.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![cfg_attr(
16    not(any(
17        feature = "state-store",
18        feature = "crypto-store",
19        feature = "event-cache-store",
20        feature = "media-store"
21    )),
22    allow(dead_code, unused_imports)
23)]
24
25mod connection;
26#[cfg(feature = "crypto-store")]
27mod crypto_store;
28mod error;
29#[cfg(feature = "event-cache-store")]
30mod event_cache_store;
31#[cfg(feature = "media-store")]
32mod media_store;
33#[cfg(feature = "state-store")]
34mod state_store;
35mod utils;
36use std::{
37    cmp::max,
38    fmt,
39    path::{Path, PathBuf},
40};
41
42use deadpool::managed::PoolConfig;
43use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
44
45#[cfg(feature = "crypto-store")]
46pub use self::crypto_store::SqliteCryptoStore;
47pub use self::error::OpenStoreError;
48#[cfg(feature = "event-cache-store")]
49pub use self::event_cache_store::SqliteEventCacheStore;
50#[cfg(feature = "media-store")]
51pub use self::media_store::SqliteMediaStore;
52#[cfg(feature = "state-store")]
53pub use self::state_store::{DATABASE_NAME as STATE_STORE_DATABASE_NAME, SqliteStateStore};
54
55#[cfg(feature = "uniffi")]
56uniffi::setup_scaffolding!();
57
58#[cfg(test)]
59matrix_sdk_test_utils::init_tracing_for_tests!();
60
61/// An enum used to store the secret that gives access to a store
62#[derive(Clone, Debug, PartialEq, Zeroize, ZeroizeOnDrop)]
63pub enum Secret {
64    // Cryptographic key used to open the store
65    Key(Zeroizing<Vec<u8>>),
66    // Passphrase used to open the store, ideally human chosen
67    PassPhrase(Zeroizing<String>),
68    // Randomly generated passphrase, for which the store caches a
69    // cheaply-derivable copy of its cipher and skips derivation on later opens
70    HighEntropyPassPhrase {
71        key: Zeroizing<Vec<u8>>,
72        #[zeroize(skip)]
73        base64_variant: Base64Variant,
74    },
75}
76
77/// Enum controlling how the high-entropy passphrase used to be created on the
78/// client side.
79///
80/// This allows us to replicate how a random key was converted into a passphrase
81/// to migrate from said passphrase to the plain key.
82#[derive(Clone, Copy, Debug, PartialEq, Eq)]
83#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))]
84pub enum Base64Variant {
85    /// Unpadded base64 was used to create the high-entropy passphrase.
86    Unpadded,
87    /// Standard padded base64 was used to create the high-entropy passphrase.
88    Padded,
89}
90
91/// A configuration structure used for opening a store.
92#[derive(Clone)]
93pub struct SqliteStoreConfig {
94    /// Path to the database, without the file name.
95    path: PathBuf,
96    /// Secret to open the store, if any
97    secret: Option<Secret>,
98    /// The pool configuration for [`deadpool`].
99    pool_config: PoolConfig,
100    /// The runtime configuration to apply when opening an SQLite connection.
101    runtime_config: RuntimeConfig,
102}
103
104impl fmt::Debug for SqliteStoreConfig {
105    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
106        formatter
107            .debug_struct("SqliteStoreConfig")
108            .field("path", &self.path)
109            .field("pool_config", &self.pool_config)
110            .field("runtime_config", &self.runtime_config)
111            .finish_non_exhaustive()
112    }
113}
114
115/// The minimum size of the connections pool.
116///
117/// We need at least 2 connections: one connection for write operations, and one
118/// connection for read operations.
119const POOL_MINIMUM_SIZE: usize = 2;
120
121impl SqliteStoreConfig {
122    /// Create a new [`SqliteStoreConfig`] with a path representing the
123    /// directory containing the store database.
124    pub fn new<P>(path: P) -> Self
125    where
126        P: AsRef<Path>,
127    {
128        Self {
129            path: path.as_ref().to_path_buf(),
130            pool_config: PoolConfig::new(max(POOL_MINIMUM_SIZE, num_cpus::get_physical() * 4)),
131            runtime_config: RuntimeConfig::default(),
132            secret: None,
133        }
134    }
135
136    /// Similar to [`SqliteStoreConfig::new`], but with defaults tailored for a
137    /// low memory usage environment.
138    ///
139    /// The following defaults are set:
140    ///
141    /// - The `pool_max_size` is set to the number of physical CPU, so one
142    ///   connection per physical thread,
143    /// - The `cache_size` is set to 500Kib,
144    /// - The `journal_size_limit` is set to 2Mib.
145    pub fn with_low_memory_config<P>(path: P) -> Self
146    where
147        P: AsRef<Path>,
148    {
149        Self::new(path)
150            // Maximum one connection per physical thread.
151            .pool_max_size(num_cpus::get_physical())
152            // Cache size is 500Kib.
153            .cache_size(500_000)
154            // Journal size limit is 2Mib.
155            .journal_size_limit(2_000_000)
156    }
157
158    /// Override the path.
159    pub fn path<P>(mut self, path: P) -> Self
160    where
161        P: AsRef<Path>,
162    {
163        self.path = path.as_ref().to_path_buf();
164        self
165    }
166
167    /// Define the passphrase if the store is encoded.
168    ///
169    /// Assumed to be possibly human-chosen, so an expensive derivation is run
170    /// over it on every open. If it is randomly generated, use
171    /// [`SqliteStoreConfig::high_entropy_passphrase`] instead.
172    pub fn passphrase(mut self, passphrase: Option<&str>) -> Self {
173        self.secret =
174            passphrase.map(|passphrase| Secret::PassPhrase(Zeroizing::new(passphrase.to_owned())));
175        self
176    }
177
178    /// Define the passphrase if the store is encoded, declaring that it was
179    /// randomly generated rather than chosen by a human.
180    ///
181    /// Do NOT use this with human-chosen passphrases, as doing so would remove
182    /// their brute-force protection.
183    ///
184    /// This migrates a passphrase-based store whose passphrase was created by
185    /// base64-encoding a randomly generated key to a key-based setup.
186    ///
187    /// Once this function has been called, [`SqliteStoreConfig::passphrase`]
188    /// can no longer be used with the passphrase.
189    ///
190    /// [`SqliteStoreConfig::key`] can be used with the original key, before it
191    /// was base64-encoded.
192    pub fn high_entropy_passphrase(
193        mut self,
194        passphrase: Option<&[u8]>,
195        base64_variant: Base64Variant,
196    ) -> Self {
197        if let Some(passphrase) = passphrase {
198            let key = Zeroizing::new(passphrase.to_vec());
199            self.secret = Some(Secret::HighEntropyPassPhrase { key, base64_variant });
200        }
201
202        self
203    }
204
205    /// Define the key if the store is encoded.
206    ///
207    /// Assumed to be high entropy so no derivation is run over it.
208    pub fn key(mut self, key: Option<&[u8]>) -> Self {
209        if let Some(key) = key {
210            let key = Zeroizing::new(key.to_vec());
211            self.secret = Some(Secret::Key(key));
212        }
213
214        self
215    }
216
217    /// Define the maximum pool size for [`deadpool`].
218    ///
219    /// See [`deadpool::managed::PoolConfig::max_size`] to learn more.
220    pub fn pool_max_size(mut self, max_size: usize) -> Self {
221        self.pool_config.max_size = max(POOL_MINIMUM_SIZE, max_size);
222        self
223    }
224
225    /// Optimize the database.
226    ///
227    /// The SQLite documentation recommends to run this regularly and after any
228    /// schema change. The easiest is to do it consistently when the store is
229    /// constructed, after eventual migrations.
230    ///
231    /// See [`PRAGMA optimize`] to learn more.
232    ///
233    /// The default value is `true`.
234    ///
235    /// [`PRAGMA optimize`]: https://www.sqlite.org/pragma.html#pragma_optimize
236    pub fn optimize(mut self, optimize: bool) -> Self {
237        self.runtime_config.optimize = optimize;
238        self
239    }
240
241    /// Define the maximum size in **bytes** the SQLite cache can use.
242    ///
243    /// See [`PRAGMA cache_size`] to learn more.
244    ///
245    /// The default value is 2Mib.
246    ///
247    /// [`PRAGMA cache_size`]: https://www.sqlite.org/pragma.html#pragma_cache_size
248    pub fn cache_size(mut self, cache_size: u32) -> Self {
249        self.runtime_config.cache_size = cache_size;
250        self
251    }
252
253    /// Limit the size of the WAL file, in **bytes**.
254    ///
255    /// By default, while the DB connections of the databases are open,
256    /// [the size of the WAL file can keep increasing][size_wal_file] depending
257    /// on the size needed for the transactions. A critical case is `VACUUM`
258    /// which basically writes the content of the DB file to the WAL file before
259    /// writing it back to the DB file, so we end up taking twice the size of
260    /// the database.
261    ///
262    /// By setting this limit, the WAL file is truncated after its content is
263    /// written to the database, if it is bigger than the limit.
264    ///
265    /// See [`PRAGMA journal_size_limit`] to learn more. The value `limit`
266    /// corresponds to `N` in `PRAGMA journal_size_limit = N`.
267    ///
268    /// The default value is 10Mib.
269    ///
270    /// [size_wal_file]: https://www.sqlite.org/wal.html#avoiding_excessively_large_wal_files
271    /// [`PRAGMA journal_size_limit`]: https://www.sqlite.org/pragma.html#pragma_journal_size_limit
272    pub fn journal_size_limit(mut self, limit: u32) -> Self {
273        self.runtime_config.journal_size_limit = limit;
274        self
275    }
276
277    /// Define how often SQLite syncs the database to the storage device.
278    ///
279    /// [`Synchronous::Normal`] is faster, but a power loss can drop the most
280    /// recent transactions. Defaults to [`Synchronous::Full`].
281    ///
282    /// See [`PRAGMA synchronous`] to learn more.
283    ///
284    /// [`PRAGMA synchronous`]: https://www.sqlite.org/pragma.html#pragma_synchronous
285    pub fn synchronous(mut self, synchronous: Synchronous) -> Self {
286        self.runtime_config.synchronous = synchronous;
287        self
288    }
289
290    /// Returns the pool configuration.
291    pub(crate) fn pool_config(&self) -> PoolConfig {
292        self.pool_config
293    }
294
295    /// Returns the runtime configuration.
296    pub(crate) fn runtime_config(&self) -> RuntimeConfig {
297        self.runtime_config
298    }
299
300    /// Build a pool of active connections to a particular database.
301    pub fn build_pool_of_connections(
302        &self,
303        database_name: &str,
304    ) -> Result<connection::Pool, connection::CreatePoolError> {
305        let path = self.path.join(database_name);
306        let manager = connection::Manager::new(path, self.runtime_config);
307
308        connection::Pool::builder(manager)
309            .config(self.pool_config)
310            .runtime(connection::RUNTIME)
311            .build()
312            .map_err(connection::CreatePoolError::Build)
313    }
314}
315
316/// How often SQLite syncs the database to the storage device.
317///
318/// See [`PRAGMA synchronous`] to learn more.
319///
320/// [`PRAGMA synchronous`]: https://www.sqlite.org/pragma.html#pragma_synchronous
321#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
322pub enum Synchronous {
323    /// SQLite never syncs. A crash of the operating system can corrupt the
324    /// database.
325    Off,
326
327    /// SQLite syncs at the most critical moments. In WAL mode, that is one
328    /// `fsync` per checkpoint.
329    Normal,
330
331    /// SQLite syncs at every transaction. In WAL mode, that is one `fsync` of
332    /// the WAL file per commit. This is the default of SQLite.
333    #[default]
334    Full,
335
336    /// Like [`Synchronous::Full`], plus one `fsync` of the directory when a
337    /// rollback journal is deleted. This has no effect in WAL mode.
338    Extra,
339}
340
341impl Synchronous {
342    fn as_str(self) -> &'static str {
343        match self {
344            Self::Off => "OFF",
345            Self::Normal => "NORMAL",
346            Self::Full => "FULL",
347            Self::Extra => "EXTRA",
348        }
349    }
350}
351
352/// This type represents values to set at runtime when a database is opened.
353///
354/// The per-connection part is applied by
355/// [`connection::Manager`] to every connection it creates. The rest is applied
356/// by [`utils::SqliteAsyncConnExt::apply_runtime_config`].
357#[derive(Clone, Copy, Debug)]
358pub(crate) struct RuntimeConfig {
359    /// If `true`, [`utils::SqliteAsyncConnExt::optimize`] will be called.
360    optimize: bool,
361
362    /// Regardless of the value, `PRAGMA cache_size` will always be set to this
363    /// value.
364    cache_size: u32,
365
366    /// Regardless of the value, `PRAGMA journal_size_limit` will always be set
367    /// to this value.
368    journal_size_limit: u32,
369
370    /// Regardless of the value, `PRAGMA synchronous` will always be set to this
371    /// value.
372    synchronous: Synchronous,
373}
374
375impl RuntimeConfig {
376    /// The pragmas to run on every new connection, as `synchronous`,
377    /// `cache_size` and `journal_size_limit` all apply to one connection only.
378    pub(crate) fn connection_pragmas(&self) -> String {
379        // `N` in `PRAGMA cache_size = -N` is expressed in kibibytes, while
380        // `cache_size` is expressed in bytes.
381        let cache_size_in_kib = self.cache_size / 1024;
382
383        format!(
384            "PRAGMA synchronous = {}; \
385             PRAGMA cache_size = -{cache_size_in_kib}; \
386             PRAGMA journal_size_limit = {};",
387            self.synchronous.as_str(),
388            self.journal_size_limit,
389        )
390    }
391}
392
393impl Default for RuntimeConfig {
394    fn default() -> Self {
395        Self {
396            // Optimize is always applied.
397            optimize: true,
398            // A cache of 2Mib.
399            cache_size: 2_000_000,
400            // A limit of 10Mib.
401            journal_size_limit: 10_000_000,
402            synchronous: Synchronous::default(),
403        }
404    }
405}
406
407#[cfg(test)]
408mod tests {
409    use std::{
410        ops::Not,
411        path::{Path, PathBuf},
412    };
413
414    use zeroize::Zeroizing;
415
416    use super::{POOL_MINIMUM_SIZE, Secret, SqliteStoreConfig, Synchronous};
417
418    #[test]
419    fn test_new() {
420        let store_config = SqliteStoreConfig::new(Path::new("foo"));
421
422        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical() * 4);
423        assert!(store_config.runtime_config.optimize);
424        assert_eq!(store_config.runtime_config.cache_size, 2_000_000);
425        assert_eq!(store_config.runtime_config.journal_size_limit, 10_000_000);
426        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Full);
427    }
428
429    #[test]
430    fn test_with_low_memory_config() {
431        let store_config = SqliteStoreConfig::with_low_memory_config(Path::new("foo"));
432
433        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical());
434        assert!(store_config.runtime_config.optimize);
435        assert_eq!(store_config.runtime_config.cache_size, 500_000);
436        assert_eq!(store_config.runtime_config.journal_size_limit, 2_000_000);
437        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Full);
438    }
439
440    #[test]
441    fn test_store_config_when_passphrase() {
442        let store_config = SqliteStoreConfig::new(Path::new("foo"))
443            .passphrase(Some("bar"))
444            .pool_max_size(42)
445            .optimize(false)
446            .cache_size(43)
447            .journal_size_limit(44)
448            .synchronous(Synchronous::Off);
449
450        assert_eq!(store_config.path, PathBuf::from("foo"));
451        assert_eq!(store_config.secret, Some(Secret::PassPhrase("bar".to_owned().into())));
452        assert_eq!(store_config.pool_config.max_size, 42);
453        assert!(store_config.runtime_config.optimize.not());
454        assert_eq!(store_config.runtime_config.cache_size, 43);
455        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
456        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Off);
457    }
458
459    #[test]
460    fn test_store_config_when_key() {
461        let store_config = SqliteStoreConfig::new(Path::new("foo"))
462            .key(Some(&[
463                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
464                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
465            ]))
466            .pool_max_size(42)
467            .optimize(false)
468            .cache_size(43)
469            .journal_size_limit(44)
470            .synchronous(Synchronous::Extra);
471
472        assert_eq!(store_config.path, PathBuf::from("foo"));
473        assert_eq!(
474            store_config.secret,
475            Some(Secret::Key(Zeroizing::new(vec![
476                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
477                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
478            ])))
479        );
480        assert_eq!(store_config.pool_config.max_size, 42);
481        assert!(store_config.runtime_config.optimize.not());
482        assert_eq!(store_config.runtime_config.cache_size, 43);
483        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
484        assert_eq!(store_config.runtime_config.synchronous, Synchronous::Extra);
485    }
486
487    #[test]
488    fn test_store_config_path() {
489        let store_config = SqliteStoreConfig::new(Path::new("foo")).path(Path::new("bar"));
490
491        assert_eq!(store_config.path, PathBuf::from("bar"));
492    }
493
494    #[test]
495    fn test_pool_size_has_a_minimum() {
496        let store_config = SqliteStoreConfig::new(Path::new("foo")).pool_max_size(1);
497
498        assert_eq!(store_config.pool_config.max_size, POOL_MINIMUM_SIZE);
499    }
500}
matrix_sdk_sqlite/lib.rs

matrix_sdk_sqlite/
lib.rs
