matrix_sdk_sqlite/
lib.rs

1// Copyright 2022 The Matrix.org Foundation C.I.C.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![cfg_attr(
16    not(any(
17        feature = "state-store",
18        feature = "crypto-store",
19        feature = "event-cache-store",
20        feature = "media-store"
21    )),
22    allow(dead_code, unused_imports)
23)]
24
25mod connection;
26#[cfg(feature = "crypto-store")]
27mod crypto_store;
28mod error;
29#[cfg(feature = "event-cache-store")]
30mod event_cache_store;
31#[cfg(feature = "media-store")]
32mod media_store;
33#[cfg(feature = "state-store")]
34mod state_store;
35mod utils;
36use std::{
37    cmp::max,
38    fmt,
39    path::{Path, PathBuf},
40};
41
42use deadpool::managed::PoolConfig;
43use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
44
45#[cfg(feature = "crypto-store")]
46pub use self::crypto_store::SqliteCryptoStore;
47pub use self::error::OpenStoreError;
48#[cfg(feature = "event-cache-store")]
49pub use self::event_cache_store::SqliteEventCacheStore;
50#[cfg(feature = "media-store")]
51pub use self::media_store::SqliteMediaStore;
52#[cfg(feature = "state-store")]
53pub use self::state_store::{DATABASE_NAME as STATE_STORE_DATABASE_NAME, SqliteStateStore};
54
55#[cfg(test)]
56matrix_sdk_test_utils::init_tracing_for_tests!();
57
58/// An enum used to store the secret that gives access to a store
59#[derive(Clone, Debug, PartialEq, Zeroize, ZeroizeOnDrop)]
60pub enum Secret {
61    // Cryptographic key used to open the store
62    Key(Box<[u8; 32]>),
63    // Passphrase used to open the store
64    PassPhrase(Zeroizing<String>),
65}
66
67/// A configuration structure used for opening a store.
68#[derive(Clone)]
69pub struct SqliteStoreConfig {
70    /// Path to the database, without the file name.
71    path: PathBuf,
72    /// Secret to open the store, if any
73    secret: Option<Secret>,
74    /// The pool configuration for [`deadpool`].
75    pool_config: PoolConfig,
76    /// The runtime configuration to apply when opening an SQLite connection.
77    runtime_config: RuntimeConfig,
78}
79
80impl fmt::Debug for SqliteStoreConfig {
81    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
82        formatter
83            .debug_struct("SqliteStoreConfig")
84            .field("path", &self.path)
85            .field("pool_config", &self.pool_config)
86            .field("runtime_config", &self.runtime_config)
87            .finish_non_exhaustive()
88    }
89}
90
91/// The minimum size of the connections pool.
92///
93/// We need at least 2 connections: one connection for write operations, and one
94/// connection for read operations.
95const POOL_MINIMUM_SIZE: usize = 2;
96
97impl SqliteStoreConfig {
98    /// Create a new [`SqliteStoreConfig`] with a path representing the
99    /// directory containing the store database.
100    pub fn new<P>(path: P) -> Self
101    where
102        P: AsRef<Path>,
103    {
104        Self {
105            path: path.as_ref().to_path_buf(),
106            pool_config: PoolConfig::new(max(POOL_MINIMUM_SIZE, num_cpus::get_physical() * 4)),
107            runtime_config: RuntimeConfig::default(),
108            secret: None,
109        }
110    }
111
112    /// Similar to [`SqliteStoreConfig::new`], but with defaults tailored for a
113    /// low memory usage environment.
114    ///
115    /// The following defaults are set:
116    ///
117    /// * The `pool_max_size` is set to the number of physical CPU, so one
118    ///   connection per physical thread,
119    /// * The `cache_size` is set to 500Kib,
120    /// * The `journal_size_limit` is set to 2Mib.
121    pub fn with_low_memory_config<P>(path: P) -> Self
122    where
123        P: AsRef<Path>,
124    {
125        Self::new(path)
126            // Maximum one connection per physical thread.
127            .pool_max_size(num_cpus::get_physical())
128            // Cache size is 500Kib.
129            .cache_size(500_000)
130            // Journal size limit is 2Mib.
131            .journal_size_limit(2_000_000)
132    }
133
134    /// Override the path.
135    pub fn path<P>(mut self, path: P) -> Self
136    where
137        P: AsRef<Path>,
138    {
139        self.path = path.as_ref().to_path_buf();
140        self
141    }
142
143    /// Define the passphrase if the store is encoded.
144    pub fn passphrase(mut self, passphrase: Option<&str>) -> Self {
145        self.secret =
146            passphrase.map(|passphrase| Secret::PassPhrase(Zeroizing::new(passphrase.to_owned())));
147        self
148    }
149
150    /// Define the key if the store is encoded.
151    pub fn key(mut self, key: Option<&[u8; 32]>) -> Self {
152        self.secret = key.map(|key| Secret::Key(Box::new(*key)));
153        self
154    }
155
156    /// Define the maximum pool size for [`deadpool`].
157    ///
158    /// See [`deadpool::managed::PoolConfig::max_size`] to learn more.
159    pub fn pool_max_size(mut self, max_size: usize) -> Self {
160        self.pool_config.max_size = max(POOL_MINIMUM_SIZE, max_size);
161        self
162    }
163
164    /// Optimize the database.
165    ///
166    /// The SQLite documentation recommends to run this regularly and after any
167    /// schema change. The easiest is to do it consistently when the store is
168    /// constructed, after eventual migrations.
169    ///
170    /// See [`PRAGMA optimize`] to learn more.
171    ///
172    /// The default value is `true`.
173    ///
174    /// [`PRAGMA optimize`]: https://www.sqlite.org/pragma.html#pragma_optimize
175    pub fn optimize(mut self, optimize: bool) -> Self {
176        self.runtime_config.optimize = optimize;
177        self
178    }
179
180    /// Define the maximum size in **bytes** the SQLite cache can use.
181    ///
182    /// See [`PRAGMA cache_size`] to learn more.
183    ///
184    /// The default value is 2Mib.
185    ///
186    /// [`PRAGMA cache_size`]: https://www.sqlite.org/pragma.html#pragma_cache_size
187    pub fn cache_size(mut self, cache_size: u32) -> Self {
188        self.runtime_config.cache_size = cache_size;
189        self
190    }
191
192    /// Limit the size of the WAL file, in **bytes**.
193    ///
194    /// By default, while the DB connections of the databases are open, [the
195    /// size of the WAL file can keep increasing][size_wal_file] depending on
196    /// the size needed for the transactions. A critical case is `VACUUM`
197    /// which basically writes the content of the DB file to the WAL file
198    /// before writing it back to the DB file, so we end up taking twice the
199    /// size of the database.
200    ///
201    /// By setting this limit, the WAL file is truncated after its content is
202    /// written to the database, if it is bigger than the limit.
203    ///
204    /// See [`PRAGMA journal_size_limit`] to learn more. The value `limit`
205    /// corresponds to `N` in `PRAGMA journal_size_limit = N`.
206    ///
207    /// The default value is 10Mib.
208    ///
209    /// [size_wal_file]: https://www.sqlite.org/wal.html#avoiding_excessively_large_wal_files
210    /// [`PRAGMA journal_size_limit`]: https://www.sqlite.org/pragma.html#pragma_journal_size_limit
211    pub fn journal_size_limit(mut self, limit: u32) -> Self {
212        self.runtime_config.journal_size_limit = limit;
213        self
214    }
215
216    /// Returns the pool configuration.
217    pub(crate) fn pool_config(&self) -> PoolConfig {
218        self.pool_config
219    }
220
221    /// Returns the runtime configuration.
222    pub(crate) fn runtime_config(&self) -> RuntimeConfig {
223        self.runtime_config
224    }
225
226    /// Build a pool of active connections to a particular database.
227    pub fn build_pool_of_connections(
228        &self,
229        database_name: &str,
230    ) -> Result<connection::Pool, connection::CreatePoolError> {
231        let path = self.path.join(database_name);
232        let manager = connection::Manager::new(path);
233
234        connection::Pool::builder(manager)
235            .config(self.pool_config)
236            .runtime(connection::RUNTIME)
237            .build()
238            .map_err(connection::CreatePoolError::Build)
239    }
240}
241
242/// This type represents values to set at runtime when a database is opened.
243///
244/// This configuration is applied by
245/// [`utils::SqliteAsyncConnExt::apply_runtime_config`].
246#[derive(Clone, Copy, Debug)]
247struct RuntimeConfig {
248    /// If `true`, [`utils::SqliteAsyncConnExt::optimize`] will be called.
249    optimize: bool,
250
251    /// Regardless of the value, [`utils::SqliteAsyncConnExt::cache_size`] will
252    /// always be called with this value.
253    cache_size: u32,
254
255    /// Regardless of the value,
256    /// [`utils::SqliteAsyncConnExt::journal_size_limit`] will always be called
257    /// with this value.
258    journal_size_limit: u32,
259}
260
261impl Default for RuntimeConfig {
262    fn default() -> Self {
263        Self {
264            // Optimize is always applied.
265            optimize: true,
266            // A cache of 2Mib.
267            cache_size: 2_000_000,
268            // A limit of 10Mib.
269            journal_size_limit: 10_000_000,
270        }
271    }
272}
273
274#[cfg(test)]
275mod tests {
276    use std::{
277        ops::Not,
278        path::{Path, PathBuf},
279    };
280
281    use super::{POOL_MINIMUM_SIZE, Secret, SqliteStoreConfig};
282
283    #[test]
284    fn test_new() {
285        let store_config = SqliteStoreConfig::new(Path::new("foo"));
286
287        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical() * 4);
288        assert!(store_config.runtime_config.optimize);
289        assert_eq!(store_config.runtime_config.cache_size, 2_000_000);
290        assert_eq!(store_config.runtime_config.journal_size_limit, 10_000_000);
291    }
292
293    #[test]
294    fn test_with_low_memory_config() {
295        let store_config = SqliteStoreConfig::with_low_memory_config(Path::new("foo"));
296
297        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical());
298        assert!(store_config.runtime_config.optimize);
299        assert_eq!(store_config.runtime_config.cache_size, 500_000);
300        assert_eq!(store_config.runtime_config.journal_size_limit, 2_000_000);
301    }
302
303    #[test]
304    fn test_store_config_when_passphrase() {
305        let store_config = SqliteStoreConfig::new(Path::new("foo"))
306            .passphrase(Some("bar"))
307            .pool_max_size(42)
308            .optimize(false)
309            .cache_size(43)
310            .journal_size_limit(44);
311
312        assert_eq!(store_config.path, PathBuf::from("foo"));
313        assert_eq!(store_config.secret, Some(Secret::PassPhrase("bar".to_owned().into())));
314        assert_eq!(store_config.pool_config.max_size, 42);
315        assert!(store_config.runtime_config.optimize.not());
316        assert_eq!(store_config.runtime_config.cache_size, 43);
317        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
318    }
319
320    #[test]
321    fn test_store_config_when_key() {
322        let store_config = SqliteStoreConfig::new(Path::new("foo"))
323            .key(Some(&[
324                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
325                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
326            ]))
327            .pool_max_size(42)
328            .optimize(false)
329            .cache_size(43)
330            .journal_size_limit(44);
331
332        assert_eq!(store_config.path, PathBuf::from("foo"));
333        assert_eq!(
334            store_config.secret,
335            Some(Secret::Key(Box::new([
336                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
337                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
338            ])))
339        );
340        assert_eq!(store_config.pool_config.max_size, 42);
341        assert!(store_config.runtime_config.optimize.not());
342        assert_eq!(store_config.runtime_config.cache_size, 43);
343        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
344    }
345
346    #[test]
347    fn test_store_config_path() {
348        let store_config = SqliteStoreConfig::new(Path::new("foo")).path(Path::new("bar"));
349
350        assert_eq!(store_config.path, PathBuf::from("bar"));
351    }
352
353    #[test]
354    fn test_pool_size_has_a_minimum() {
355        let store_config = SqliteStoreConfig::new(Path::new("foo")).pool_max_size(1);
356
357        assert_eq!(store_config.pool_config.max_size, POOL_MINIMUM_SIZE);
358    }
359}
matrix_sdk_sqlite/lib.rs

matrix_sdk_sqlite/
lib.rs
