matrix_sdk/search_index/
mod.rs

1// Copyright 2025 The Matrix.org Foundation C.I.C.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The search index is an abstraction layer in the matrix-sdk for the
16//! matrix-sdk-search crate. It provides a [`SearchIndex`] which wraps
17//! multiple [`RoomIndex`].
18
19use std::{
20    collections::hash_map::HashMap,
21    num::NonZeroUsize,
22    path::{Path, PathBuf},
23    sync::Arc,
24    thread::available_parallelism,
25};
26
27use futures_util::{StreamExt as _, future::join_all, stream};
28use matrix_sdk_base::{
29    check_validity_of_replacement_events, deserialized_responses::TimelineEvent,
30};
31use matrix_sdk_search::{
32    error::IndexError,
33    index::{
34        IndexableEvent, RoomIndex, RoomIndexOperation, SearchResult, builder::RoomIndexBuilder,
35    },
36};
37use ruma::{
38    EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, RoomId,
39    events::{
40        AnySyncMessageLikeEvent, AnySyncTimelineEvent,
41        poll::{
42            start::SyncPollStartEvent,
43            unstable_start::{SyncUnstablePollStartEvent, UnstablePollStartEventContent},
44        },
45        room::{
46            message::{MessageType, OriginalSyncRoomMessageEvent, Relation, SyncRoomMessageEvent},
47            redaction::SyncRoomRedactionEvent,
48        },
49        sticker::SyncStickerEvent,
50    },
51    room_version_rules::RedactionRules,
52};
53use tokio::{
54    sync::{Mutex, MutexGuard},
55    task::spawn_blocking,
56};
57use tracing::{debug, warn};
58
59use crate::event_cache::RoomEventCache;
60
61type Key = String;
62
63/// Marker file, in the search index directory, telling that the events saved in
64/// the event cache store have been indexed. Bump it when the index format
65/// changes, so the events get indexed again.
66pub(crate) const REINDEX_MARKER: &str = "reindexed-v1";
67
68/// Type of location to store [`RoomIndex`]
69#[derive(Clone, Debug)]
70pub enum SearchIndexStoreKind {
71    /// Store unencrypted in file system folder
72    UnencryptedDirectory(PathBuf),
73    /// Store encrypted, with the given high-entropy key, in file system folder
74    EncryptedDirectory(PathBuf, Key),
75    /// Store in memory
76    InMemory,
77}
78
79/// Object that handles inteeraction with [`RoomIndex`]'s for search
80#[derive(Clone, Debug)]
81pub struct SearchIndex {
82    /// HashMap that links each joined room to its RoomIndex
83    room_indexes: Arc<Mutex<HashMap<OwnedRoomId, RoomIndex>>>,
84
85    /// Base directory that stores the directories for each RoomIndex
86    search_index_store_kind: SearchIndexStoreKind,
87}
88
89impl SearchIndex {
90    /// Create a new [`SearchIndex`]
91    pub fn new(
92        room_indexes: Arc<Mutex<HashMap<OwnedRoomId, RoomIndex>>>,
93        search_index_store_kind: SearchIndexStoreKind,
94    ) -> Self {
95        Self { room_indexes, search_index_store_kind }
96    }
97
98    /// The directory the room indexes are stored in, if they are stored on
99    /// disk.
100    pub(crate) fn directory(&self) -> Option<&Path> {
101        match &self.search_index_store_kind {
102            SearchIndexStoreKind::UnencryptedDirectory(path)
103            | SearchIndexStoreKind::EncryptedDirectory(path, _) => Some(path),
104            SearchIndexStoreKind::InMemory => None,
105        }
106    }
107
108    /// The disk size of the room indexes, if they are stored on disk.
109    pub(crate) fn size(&self) -> Option<usize> {
110        fn dir_size(path: &Path) -> std::io::Result<u64> {
111            let mut size = 0;
112            for entry in std::fs::read_dir(path)? {
113                let entry = entry?;
114                let metadata = entry.metadata()?;
115                size += if metadata.is_dir() { dir_size(&entry.path())? } else { metadata.len() };
116            }
117            Ok(size)
118        }
119
120        let size = dir_size(self.directory()?).ok()?;
121        usize::try_from(size).ok()
122    }
123
124    /// Delete all the room indexes, from memory and from disk, along with the
125    /// reindex marker. Any other file in the directory is kept, so it can be
126    /// shared with other data, like the stores.
127    pub async fn clear(&self) -> std::io::Result<()> {
128        // Hold the lock, so no index is opened or written to meanwhile.
129        let mut room_indexes = self.room_indexes.lock().await;
130
131        // Drop the open indexes, so their files are closed before deletion.
132        room_indexes.clear();
133
134        let Some(directory) = self.directory() else { return Ok(()) };
135
136        let entries = match std::fs::read_dir(directory) {
137            Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
138            entries => entries?,
139        };
140
141        for entry in entries {
142            let entry = entry?;
143
144            if entry.file_name() == REINDEX_MARKER {
145                std::fs::remove_file(entry.path())?;
146            } else if entry.file_type()?.is_dir()
147                && entry.file_name().to_str().is_some_and(|name| RoomId::parse(name).is_ok())
148            {
149                // Room indexes are stored in directories named after their
150                // room.
151                std::fs::remove_dir_all(entry.path())?;
152            }
153        }
154
155        Ok(())
156    }
157
158    /// Acquire [`SearchIndexGuard`] for this [`SearchIndex`].
159    pub async fn lock(&self) -> SearchIndexGuard<'_> {
160        SearchIndexGuard {
161            index_map: self.room_indexes.lock().await,
162            search_index_store_kind: &self.search_index_store_kind,
163        }
164    }
165
166    /// Open the indexes of the given rooms that exist on disk but aren't open
167    /// yet, concurrently and without holding the lock, instead of one by one
168    /// when they are first searched.
169    pub(crate) async fn open_indexes(&self, room_ids: impl Iterator<Item = OwnedRoomId>) {
170        let to_open: Vec<_> = {
171            let guard = self.lock().await;
172            room_ids
173                .filter(|room_id| {
174                    !guard.index_map.contains_key(room_id) && guard.has_index(room_id)
175                })
176                .collect()
177        };
178
179        let concurrency = available_parallelism().map_or(1, NonZeroUsize::get);
180
181        let mut opened = stream::iter(to_open)
182            .map(|room_id| {
183                let store_kind = self.search_index_store_kind.clone();
184                spawn_blocking(move || {
185                    let index = create_index(&store_kind, &room_id)
186                        .and_then(|index| index.warm_up().map(|()| index));
187                    (room_id, index)
188                })
189            })
190            .buffer_unordered(concurrency);
191
192        while let Some(task) = opened.next().await {
193            match task.expect("Task join error") {
194                // Keep the index opened meanwhile by the indexer, if any.
195                (room_id, Ok(index)) => {
196                    self.room_indexes.lock().await.entry(room_id).or_insert(index);
197                }
198                // The search will try again and report the error.
199                (room_id, Err(err)) => warn!(%room_id, "Failed to open the search index: {err}"),
200            }
201        }
202    }
203
204    /// Same as [`SearchIndexGuard::bulk_handle_timeline_event`], but only
205    /// locks the index once the events are parsed: parsing waits on the event
206    /// cache, and searches need the lock meanwhile.
207    pub(crate) async fn bulk_handle_timeline_event<T>(
208        &self,
209        events: T,
210        room_cache: &RoomEventCache,
211        room_id: &RoomId,
212        redaction_rules: &RedactionRules,
213    ) -> Result<(), IndexError>
214    where
215        T: Iterator<Item = TimelineEvent>,
216    {
217        let operations = parse_timeline_events(events, room_cache, redaction_rules).await;
218        self.lock().await.bulk_execute(operations, room_id)
219    }
220}
221
222/// Object that represents an acquired [`SearchIndex`].
223#[derive(Debug)]
224pub struct SearchIndexGuard<'a> {
225    /// Guard around the [`RoomIndex`] map
226    index_map: MutexGuard<'a, HashMap<OwnedRoomId, RoomIndex>>,
227
228    /// Base directory that stores the directories for each RoomIndex
229    search_index_store_kind: &'a SearchIndexStoreKind,
230}
231
232fn create_index(
233    search_index_store_kind: &SearchIndexStoreKind,
234    room_id: &RoomId,
235) -> Result<RoomIndex, IndexError> {
236    let index = match search_index_store_kind {
237        SearchIndexStoreKind::UnencryptedDirectory(path) => {
238            RoomIndexBuilder::new_on_disk(path.to_path_buf(), room_id).unencrypted().build()?
239        }
240        SearchIndexStoreKind::EncryptedDirectory(path, key) => {
241            RoomIndexBuilder::new_on_disk(path.to_path_buf(), room_id).encrypted(key).build()?
242        }
243        SearchIndexStoreKind::InMemory => RoomIndexBuilder::new_in_memory(room_id).build(),
244    };
245    Ok(index)
246}
247
248impl SearchIndexGuard<'_> {
249    /// Handle a [`RoomIndexOperation`] in the [`RoomIndex`] of a given
250    /// [`RoomId`]
251    ///
252    /// This which will add/remove/edit an event in the index based on the event
253    /// type.
254    ///
255    /// Prefer [`SearchIndexGuard::bulk_execute`] for multiple operations.
256    pub(crate) fn execute(
257        &mut self,
258        operation: RoomIndexOperation,
259        room_id: &RoomId,
260    ) -> Result<(), IndexError> {
261        if !self.index_map.contains_key(room_id) {
262            let index = create_index(self.search_index_store_kind, room_id)?;
263            self.index_map.insert(room_id.to_owned(), index);
264        }
265
266        let index = self.index_map.get_mut(room_id).expect("index should exist");
267
268        index.execute(operation)
269    }
270
271    /// Whether a [`RoomIndex`] exists for this room, in memory or on disk.
272    fn has_index(&self, room_id: &RoomId) -> bool {
273        self.index_map.contains_key(room_id)
274            || match self.search_index_store_kind {
275                SearchIndexStoreKind::UnencryptedDirectory(path)
276                | SearchIndexStoreKind::EncryptedDirectory(path, _) => {
277                    path.join(room_id.as_str()).exists()
278                }
279                SearchIndexStoreKind::InMemory => false,
280            }
281    }
282
283    /// Handle a [`RoomIndexOperation`] in the [`RoomIndex`] of a given
284    /// [`RoomId`]
285    ///
286    /// This which will add/remove/edit an event in the index based on the event
287    /// type.
288    pub(crate) fn bulk_execute(
289        &mut self,
290        operations: Vec<RoomIndexOperation>,
291        room_id: &RoomId,
292    ) -> Result<(), IndexError> {
293        // Don't create an index (which is costly) for nothing.
294        if operations.is_empty() {
295            return Ok(());
296        }
297
298        if !self.index_map.contains_key(room_id) {
299            let index = create_index(self.search_index_store_kind, room_id)?;
300            self.index_map.insert(room_id.to_owned(), index);
301        }
302
303        let index = self.index_map.get_mut(room_id).expect("index should exist");
304
305        index.bulk_execute(operations)
306    }
307
308    /// Search a [`Room`]'s index for the query and return at most
309    /// max_number_of_results results.
310    pub(crate) fn search(
311        &mut self,
312        query: &str,
313        max_number_of_results: usize,
314        pagination_offset: Option<usize>,
315        room_id: &RoomId,
316    ) -> Result<SearchResult, IndexError> {
317        // Nothing was ever indexed for this room: don't create an index (which
318        // is costly) only to find it empty.
319        if !self.has_index(room_id) {
320            return Ok(SearchResult { total_count: 0, events: Vec::new() });
321        }
322
323        if !self.index_map.contains_key(room_id) {
324            let index = create_index(self.search_index_store_kind, room_id)?;
325            self.index_map.insert(room_id.to_owned(), index);
326        }
327
328        let index = self.index_map.get_mut(room_id).expect("index should exist");
329
330        index.search(query, max_number_of_results, pagination_offset)
331    }
332
333    /// Given a [`TimelineEvent`] this function will derive a
334    /// [`RoomIndexOperation`], if it should be handled, and execute it;
335    /// returning the result.
336    ///
337    /// Prefer [`SearchIndexGuard::bulk_handle_timeline_event`] for multiple
338    /// events.
339    pub async fn handle_timeline_event(
340        &mut self,
341        event: TimelineEvent,
342        room_cache: &RoomEventCache,
343        room_id: &RoomId,
344        redaction_rules: &RedactionRules,
345    ) -> Result<(), IndexError> {
346        if let Some(index_operation) =
347            parse_timeline_event(room_cache, event, redaction_rules).await
348        {
349            self.execute(index_operation, room_id)
350        } else {
351            Ok(())
352        }
353    }
354
355    /// Run [`SearchIndexGuard::handle_timeline_event`] for multiple
356    /// [`TimelineEvent`].
357    pub async fn bulk_handle_timeline_event<T>(
358        &mut self,
359        events: T,
360        room_cache: &RoomEventCache,
361        room_id: &RoomId,
362        redaction_rules: &RedactionRules,
363    ) -> Result<(), IndexError>
364    where
365        T: Iterator<Item = TimelineEvent>,
366    {
367        let operations = parse_timeline_events(events, room_cache, redaction_rules).await;
368        self.bulk_execute(operations, room_id)
369    }
370}
371
372/// Derive the [`RoomIndexOperation`]s to execute for the given events.
373async fn parse_timeline_events(
374    events: impl Iterator<Item = TimelineEvent>,
375    room_cache: &RoomEventCache,
376    redaction_rules: &RedactionRules,
377) -> Vec<RoomIndexOperation> {
378    let futures = events.map(|ev| parse_timeline_event(room_cache, ev, redaction_rules));
379    join_all(futures).await.into_iter().flatten().collect()
380}
381
382/// Given an event id this function returns the most recent edit on said event
383/// or the event itself if there are no edits.
384async fn get_most_recent_edit(
385    cache: &RoomEventCache,
386    original: &EventId,
387) -> Option<OriginalSyncRoomMessageEvent> {
388    use ruma::events::{AnySyncTimelineEvent, relation::RelationType};
389
390    let Ok(Some((original_ev, related))) =
391        cache.find_event_with_relations(original, Some(vec![RelationType::Replacement])).await
392    else {
393        debug!("Couldn't find relations for {}", original);
394        return None;
395    };
396
397    // Only index valid replacements (matching sender, type, etc.); otherwise
398    // anyone could rewrite another user's indexed message. Fall back to the
399    // original event when there is no valid edit.
400    let latest = related
401        .iter()
402        .rev()
403        .find(|edit| {
404            check_validity_of_replacement_events(
405                original_ev.raw(),
406                original_ev.encryption_info().map(|info| &**info),
407                edit.raw(),
408                edit.encryption_info().map(|info| &**info),
409            )
410            .is_ok()
411        })
412        .unwrap_or(&original_ev);
413
414    match latest.raw().deserialize() {
415        Ok(AnySyncTimelineEvent::MessageLike(AnySyncMessageLikeEvent::RoomMessage(latest))) => {
416            latest.as_original().cloned()
417        }
418        _ => None,
419    }
420}
421
422/// Indexable text for a media message: its filename plus any caption.
423fn media_body(filename: &str, caption: Option<&str>) -> String {
424    match caption {
425        Some(caption) => format!("{filename} {caption}"),
426        None => filename.to_owned(),
427    }
428}
429
430/// Extract the searchable text from a room message, or `None` if its type
431/// carries no text to index.
432fn room_message_body(msgtype: &MessageType) -> Option<String> {
433    match msgtype {
434        MessageType::Text(content) => Some(content.body.clone()),
435        MessageType::Emote(content) => Some(content.body.clone()),
436        MessageType::Notice(content) => Some(content.body.clone()),
437        MessageType::ServerNotice(content) => Some(content.body.clone()),
438        MessageType::Location(content) => Some(content.body.clone()),
439        MessageType::Image(content) => Some(media_body(content.filename(), content.caption())),
440        MessageType::Video(content) => Some(media_body(content.filename(), content.caption())),
441        MessageType::Audio(content) => Some(media_body(content.filename(), content.caption())),
442        MessageType::File(content) => Some(media_body(content.filename(), content.caption())),
443        _ => None,
444    }
445}
446
447/// Build an [`IndexableEvent`] from a room message, or `None` if its type
448/// carries no searchable text.
449fn indexable_from_room_message(
450    event: &OriginalSyncRoomMessageEvent,
451    timestamp: Option<MilliSecondsSinceUnixEpoch>,
452) -> Option<IndexableEvent> {
453    let body = room_message_body(&event.content.msgtype)?;
454    let original_event_id = match &event.content.relates_to {
455        Some(Relation::Replacement(replacement)) => replacement.event_id.clone(),
456        _ => event.event_id.clone(),
457    };
458
459    Some(IndexableEvent::new(
460        event.event_id.clone(),
461        original_event_id,
462        event.sender.clone(),
463        timestamp,
464        body,
465    ))
466}
467
468/// If the given [`OriginalSyncRoomMessageEvent`] is an edit we make an
469/// [`RoomIndexOperation::Edit`] with the new most recent version of the
470/// original.
471async fn handle_possible_edit(
472    event: &OriginalSyncRoomMessageEvent,
473    timestamp: Option<MilliSecondsSinceUnixEpoch>,
474    cache: &RoomEventCache,
475) -> Option<RoomIndexOperation> {
476    if let Some(Relation::Replacement(replacement_data)) = &event.content.relates_to {
477        if let Some(recent) = get_most_recent_edit(cache, &replacement_data.event_id).await {
478            return Some(
479                indexable_from_room_message(&recent, timestamp).map_or(
480                    RoomIndexOperation::Noop,
481                    |indexable| {
482                        RoomIndexOperation::Edit(replacement_data.event_id.clone(), indexable)
483                    },
484                ),
485            );
486        } else {
487            return Some(RoomIndexOperation::Noop);
488        }
489    }
490    None
491}
492
493/// Return a [`RoomIndexOperation::Edit`] or [`RoomIndexOperation::Add`]
494/// depending on the message.
495async fn handle_room_message(
496    event: SyncRoomMessageEvent,
497    timestamp: Option<MilliSecondsSinceUnixEpoch>,
498    cache: &RoomEventCache,
499) -> Option<RoomIndexOperation> {
500    if let Some(event) = event.as_original() {
501        return handle_possible_edit(event, timestamp, cache).await.or(get_most_recent_edit(
502            cache,
503            &event.event_id,
504        )
505        .await
506        .and_then(|recent| {
507            indexable_from_room_message(&recent, timestamp).map(RoomIndexOperation::Add)
508        }));
509    }
510    None
511}
512
513/// Return a [`RoomIndexOperation`] removing a redacted event from the index, or
514/// re-adding the most recent remaining version if an edit was redacted.
515async fn handle_room_redaction(
516    event: SyncRoomRedactionEvent,
517    timestamp: Option<MilliSecondsSinceUnixEpoch>,
518    cache: &RoomEventCache,
519    rules: &RedactionRules,
520) -> Option<RoomIndexOperation> {
521    let redacted_event_id = event.redacts(rules)?;
522
523    // If the redacted event was a room message edit, re-add the most recent
524    // remaining version instead of just removing it.
525    if let Ok(Some(redacted_event)) = cache.find_event(redacted_event_id).await
526        && let Ok(AnySyncTimelineEvent::MessageLike(AnySyncMessageLikeEvent::RoomMessage(
527            redacted_event,
528        ))) = redacted_event.raw().deserialize()
529        && let Some(redacted_event) = redacted_event.as_original()
530        && let Some(operation) = handle_possible_edit(redacted_event, timestamp, cache).await
531    {
532        return Some(operation);
533    }
534
535    // Otherwise remove the redacted event from the index. This covers plain
536    // messages, stickers and polls.
537    Some(RoomIndexOperation::Remove(redacted_event_id.to_owned()))
538}
539
540/// Return a [`RoomIndexOperation::Add`] indexing a sticker's descriptive text.
541fn handle_sticker(
542    event: SyncStickerEvent,
543    timestamp: Option<MilliSecondsSinceUnixEpoch>,
544) -> Option<RoomIndexOperation> {
545    let event = event.as_original()?;
546
547    Some(RoomIndexOperation::Add(IndexableEvent::new(
548        event.event_id.clone(),
549        event.event_id.clone(),
550        event.sender.clone(),
551        timestamp,
552        event.content.body.clone(),
553    )))
554}
555
556/// Return a [`RoomIndexOperation::Add`] indexing an unstable poll's question
557/// and answers.
558///
559/// ponytail: only indexes the initial `New` poll — edits (`Replacement`) and
560/// poll ends are ignored. Add edit handling if editing a poll needs to update
561/// search results.
562fn handle_unstable_poll_start(
563    event: SyncUnstablePollStartEvent,
564    timestamp: Option<MilliSecondsSinceUnixEpoch>,
565) -> Option<RoomIndexOperation> {
566    let event = event.as_original()?;
567
568    let UnstablePollStartEventContent::New(content) = &event.content else {
569        return None;
570    };
571
572    let block = &content.poll_start;
573    let mut body = block.question.text.clone();
574    for answer in block.answers.iter() {
575        body.push(' ');
576        body.push_str(&answer.text);
577    }
578
579    Some(RoomIndexOperation::Add(IndexableEvent::new(
580        event.event_id.clone(),
581        event.event_id.clone(),
582        event.sender.clone(),
583        timestamp,
584        body,
585    )))
586}
587
588/// Return a [`RoomIndexOperation::Add`] indexing a stable poll's question and
589/// answers.
590///
591/// ponytail: like [`handle_unstable_poll_start`], edits and poll ends are
592/// ignored — only the initial poll is indexed.
593fn handle_poll_start(
594    event: SyncPollStartEvent,
595    timestamp: Option<MilliSecondsSinceUnixEpoch>,
596) -> Option<RoomIndexOperation> {
597    let event = event.as_original()?;
598
599    // Skip poll edits, matching the unstable poll handling.
600    if let Some(Relation::Replacement(_)) = &event.content.relates_to {
601        return None;
602    }
603
604    let block = &event.content.poll;
605    let mut body = block.question.text.find_plain()?.to_owned();
606    for answer in block.answers.iter() {
607        if let Some(text) = answer.text.find_plain() {
608            body.push(' ');
609            body.push_str(text);
610        }
611    }
612
613    Some(RoomIndexOperation::Add(IndexableEvent::new(
614        event.event_id.clone(),
615        event.event_id.clone(),
616        event.sender.clone(),
617        timestamp,
618        body,
619    )))
620}
621
622/// Prepare a [`TimelineEvent`] into a [`RoomIndexOperation`] for search
623/// indexing.
624async fn parse_timeline_event(
625    cache: &RoomEventCache,
626    event: TimelineEvent,
627    redaction_rules: &RedactionRules,
628) -> Option<RoomIndexOperation> {
629    use ruma::events::AnySyncTimelineEvent;
630
631    if event.kind.is_utd() {
632        return None;
633    }
634
635    let timestamp = event.timestamp();
636
637    match event.raw().deserialize() {
638        Ok(event) => match event {
639            AnySyncTimelineEvent::MessageLike(event) => match event {
640                AnySyncMessageLikeEvent::RoomMessage(event) => {
641                    handle_room_message(event, timestamp, cache).await
642                }
643                AnySyncMessageLikeEvent::RoomRedaction(event) => {
644                    handle_room_redaction(event, timestamp, cache, redaction_rules).await
645                }
646                AnySyncMessageLikeEvent::Sticker(event) => handle_sticker(event, timestamp),
647                AnySyncMessageLikeEvent::PollStart(event) => handle_poll_start(event, timestamp),
648                AnySyncMessageLikeEvent::UnstablePollStart(event) => {
649                    handle_unstable_poll_start(event, timestamp)
650                }
651                _ => None,
652            },
653            AnySyncTimelineEvent::State(_) => None,
654        },
655
656        Err(e) => {
657            warn!("failed to parse event: {e:?}");
658            None
659        }
660    }
661}
662
663#[cfg(test)]
664mod tests {
665    use matrix_sdk_test::{JoinedRoomBuilder, async_test, event_factory::EventFactory};
666    use ruma::{
667        event_id, events::room::message::RoomMessageEventContentWithoutRelation, room_id, user_id,
668    };
669
670    use crate::test_utils::mocks::MatrixMockServer;
671
672    #[cfg(feature = "experimental-search")]
673    #[async_test]
674    async fn test_search_index_size() {
675        use super::SearchIndexStoreKind;
676
677        let dir = tempfile::tempdir().unwrap();
678        let mock_server = MatrixMockServer::new().await;
679        let client = mock_server
680            .client_builder()
681            .on_builder(|builder| {
682                builder.search_index_store(SearchIndexStoreKind::UnencryptedDirectory(
683                    dir.path().join("search"),
684                ))
685            })
686            .build()
687            .await;
688
689        // Nothing indexed yet: the directory doesn't even exist.
690        assert_eq!(client.get_store_sizes().await.unwrap().search_index, None);
691
692        let room_id = room_id!("!room_id:localhost");
693        mock_server
694            .sync_room(
695                &client,
696                JoinedRoomBuilder::new(room_id).add_timeline_event(
697                    EventFactory::new()
698                        .text_msg("this is a sentence")
699                        .room(room_id)
700                        .sender(user_id!("@user_id:localhost")),
701                ),
702            )
703            .await;
704
705        // Let the indexer process the event.
706        for _ in 0..50 {
707            if client.get_store_sizes().await.unwrap().search_index.is_some_and(|size| size > 0) {
708                return;
709            }
710            crate::sleep::sleep(std::time::Duration::from_millis(100)).await;
711        }
712        panic!("the search index size was never reported");
713    }
714
715    #[cfg(feature = "experimental-search")]
716    #[async_test]
717    async fn test_no_index_is_created_for_nothing() {
718        use super::{SearchIndex, SearchIndexStoreKind};
719
720        let dir = tempfile::tempdir().unwrap();
721        let search_index = SearchIndex::new(
722            Default::default(),
723            SearchIndexStoreKind::UnencryptedDirectory(dir.path().to_path_buf()),
724        );
725        let room_id = room_id!("!room_id:localhost");
726        let mut guard = search_index.lock().await;
727
728        // Searching a room that was never indexed finds nothing…
729        let result = guard.search("sentence", 10, None, room_id).unwrap();
730        assert_eq!(result.total_count, 0);
731        assert!(result.events.is_empty());
732
733        // … and so does an empty batch of operations: none creates an index.
734        guard.bulk_execute(Vec::new(), room_id).unwrap();
735
736        assert!(!dir.path().join(room_id.as_str()).exists());
737    }
738
739    #[cfg(feature = "experimental-search")]
740    #[async_test]
741    async fn test_clear_deletes_the_room_indexes_only() {
742        use super::{REINDEX_MARKER, SearchIndexStoreKind};
743
744        let dir = tempfile::tempdir().unwrap();
745        let mock_server = MatrixMockServer::new().await;
746        let client = mock_server
747            .client_builder()
748            .on_builder(|builder| {
749                builder.search_index_store(SearchIndexStoreKind::UnencryptedDirectory(
750                    dir.path().to_path_buf(),
751                ))
752            })
753            .build()
754            .await;
755
756        let room_id = room_id!("!room_id:localhost");
757        mock_server
758            .sync_room(
759                &client,
760                JoinedRoomBuilder::new(room_id).add_timeline_event(
761                    EventFactory::new()
762                        .text_msg("this is a sentence")
763                        .room(room_id)
764                        .sender(user_id!("@user_id:localhost")),
765                ),
766            )
767            .await;
768
769        // Let the indexer open the room index.
770        let room_index = dir.path().join(room_id.as_str());
771        for _ in 0..50 {
772            if room_index.exists() {
773                break;
774            }
775            crate::sleep::sleep(std::time::Duration::from_millis(100)).await;
776        }
777        assert!(room_index.exists());
778
779        // The directory can be shared with other data.
780        let other_file = dir.path().join("matrix-sdk-crypto.sqlite3");
781        std::fs::write(&other_file, b"").unwrap();
782        std::fs::write(dir.path().join(REINDEX_MARKER), b"").unwrap();
783
784        client.search_index().clear().await.unwrap();
785
786        assert!(client.search_index().lock().await.index_map.is_empty());
787        assert!(!room_index.exists());
788        assert!(!dir.path().join(REINDEX_MARKER).exists());
789        assert!(other_file.exists());
790    }
791
792    #[cfg(feature = "experimental-search")]
793    #[async_test]
794    async fn test_sync_message_is_indexed() {
795        let mock_server = MatrixMockServer::new().await;
796        let client = mock_server.client_builder().build().await;
797
798        let room_id = room_id!("!room_id:localhost");
799        let event_id = event_id!("$event_id:localost");
800        let user_id = user_id!("@user_id:localost");
801
802        let event_factory = EventFactory::new();
803        let room = mock_server
804            .sync_room(
805                &client,
806                JoinedRoomBuilder::new(room_id).add_timeline_bulk(vec![
807                    event_factory
808                        .text_msg("this is a sentence")
809                        .event_id(event_id)
810                        .sender(user_id)
811                        .into_raw_sync(),
812                ]),
813            )
814            .await;
815
816        let response = room.search("this", 5, None).await.expect("search should have 1 result");
817
818        assert_eq!(response.events.len(), 1, "unexpected numbers of responses: {response:?}");
819        assert_eq!(response.events[0].1, event_id, "event id doesn't match: {response:?}");
820    }
821
822    #[cfg(feature = "experimental-search")]
823    #[async_test]
824    async fn test_sync_media_message_is_indexed() {
825        use ruma::owned_mxc_uri;
826
827        let mock_server = MatrixMockServer::new().await;
828        let client = mock_server.client_builder().build().await;
829
830        let room_id = room_id!("!room_id:localhost");
831        let image_id = event_id!("$image_id:localhost");
832        let file_id = event_id!("$file_id:localhost");
833        let user_id = user_id!("@user_id:localhost");
834
835        let f = EventFactory::new();
836        let room = mock_server
837            .sync_room(
838                &client,
839                JoinedRoomBuilder::new(room_id).add_timeline_bulk(vec![
840                    f.image("holiday_beach.jpg".to_owned(), owned_mxc_uri!("mxc://localhost/1"))
841                        .caption(Some("sunset over the ocean".to_owned()), None)
842                        .event_id(image_id)
843                        .sender(user_id)
844                        .into_raw_sync(),
845                    f.image("quarterly_report.pdf".to_owned(), owned_mxc_uri!("mxc://localhost/2"))
846                        .event_id(file_id)
847                        .sender(user_id)
848                        .into_raw_sync(),
849                ]),
850            )
851            .await;
852
853        // The caption is indexed.
854        let response = room.search("sunset", 5, None).await.unwrap();
855        assert_eq!(response.events.len(), 1, "unexpected results for caption search: {response:?}");
856        assert_eq!(response.events[0].1, image_id, "event id doesn't match: {response:?}");
857
858        // The filename is indexed.
859        let response = room.search("holiday_beach", 5, None).await.unwrap();
860        assert_eq!(
861            response.events.len(),
862            1,
863            "unexpected results for filename search: {response:?}"
864        );
865        assert_eq!(response.events[0].1, image_id, "event id doesn't match: {response:?}");
866
867        // A media message without a caption still indexes its filename.
868        let response = room.search("quarterly_report", 5, None).await.unwrap();
869        assert_eq!(
870            response.events.len(),
871            1,
872            "unexpected results for filename search: {response:?}"
873        );
874        assert_eq!(response.events[0].1, file_id, "event id doesn't match: {response:?}");
875    }
876
877    #[cfg(feature = "experimental-search")]
878    #[async_test]
879    async fn test_sync_sticker_and_poll_are_indexed() {
880        use ruma::{events::room::ImageInfo, owned_mxc_uri};
881
882        let mock_server = MatrixMockServer::new().await;
883        let client = mock_server.client_builder().build().await;
884
885        let room_id = room_id!("!room_id:localhost");
886        let sticker_id = event_id!("$sticker_id:localhost");
887        let poll_id = event_id!("$poll_id:localhost");
888        let user_id = user_id!("@user_id:localhost");
889
890        let f = EventFactory::new().room(room_id).sender(user_id);
891        let room = mock_server
892            .sync_room(
893                &client,
894                JoinedRoomBuilder::new(room_id).add_timeline_bulk(vec![
895                    f.sticker(
896                        "a waving cat",
897                        ImageInfo::new(),
898                        owned_mxc_uri!("mxc://localhost/1"),
899                    )
900                    .event_id(sticker_id)
901                    .into_raw_sync(),
902                    f.poll_start("fallback", "favourite cheese?", vec!["comté", "gruyère"])
903                        .event_id(poll_id)
904                        .into_raw_sync(),
905                ]),
906            )
907            .await;
908
909        // The sticker's description is indexed.
910        let response = room.search("waving", 5, None).await.unwrap();
911        assert_eq!(response.events.len(), 1, "unexpected results for sticker search: {response:?}");
912        assert_eq!(response.events[0].1, sticker_id, "event id doesn't match: {response:?}");
913
914        // The poll question is indexed.
915        let response = room.search("cheese", 5, None).await.unwrap();
916        assert_eq!(
917            response.events.len(),
918            1,
919            "unexpected results for poll question search: {response:?}"
920        );
921        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
922
923        // The poll answers are indexed.
924        let response = room.search("gruyère", 5, None).await.unwrap();
925        assert_eq!(
926            response.events.len(),
927            1,
928            "unexpected results for poll answer search: {response:?}"
929        );
930        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
931    }
932
933    #[cfg(feature = "experimental-search")]
934    #[async_test]
935    async fn test_sync_stable_poll_is_indexed() {
936        use ruma::events::{
937            message::TextContentBlock,
938            poll::start::{PollAnswer, PollAnswers, PollContentBlock, PollStartEventContent},
939        };
940
941        let mock_server = MatrixMockServer::new().await;
942        let client = mock_server.client_builder().build().await;
943
944        let room_id = room_id!("!room_id:localhost");
945        let poll_id = event_id!("$stable_poll_id:localhost");
946        let user_id = user_id!("@user_id:localhost");
947
948        let answers: PollAnswers = vec![
949            PollAnswer::new("0".to_owned(), TextContentBlock::plain("comté")),
950            PollAnswer::new("1".to_owned(), TextContentBlock::plain("gruyère")),
951        ]
952        .try_into()
953        .unwrap();
954        let poll = PollContentBlock::new(TextContentBlock::plain("favourite cheese?"), answers);
955        let content = PollStartEventContent::new(TextContentBlock::plain("fallback"), poll);
956
957        let f = EventFactory::new().room(room_id).sender(user_id);
958        let room = mock_server
959            .sync_room(
960                &client,
961                JoinedRoomBuilder::new(room_id)
962                    .add_timeline_bulk(vec![f.event(content).event_id(poll_id).into_raw_sync()]),
963            )
964            .await;
965
966        // The poll question is indexed.
967        let response = room.search("cheese", 5, None).await.unwrap();
968        assert_eq!(
969            response.events.len(),
970            1,
971            "unexpected results for poll question search: {response:?}"
972        );
973        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
974
975        // The poll answers are indexed.
976        let response = room.search("gruyère", 5, None).await.unwrap();
977        assert_eq!(
978            response.events.len(),
979            1,
980            "unexpected results for poll answer search: {response:?}"
981        );
982        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
983    }
984
985    #[cfg(feature = "experimental-search")]
986    #[async_test]
987    async fn test_search_index_edit_ordering() {
988        let room_id = room_id!("!room_id:localhost");
989        let dummy_id = event_id!("$dummy");
990        let edit1_id = event_id!("$edit1");
991        let edit2_id = event_id!("$edit2");
992        let edit3_id = event_id!("$edit3");
993        let original_id = event_id!("$original");
994
995        let server = MatrixMockServer::new().await;
996        let client = server.client_builder().build().await;
997
998        let room = server.sync_joined_room(&client, room_id).await;
999
1000        let f = EventFactory::new().room(room_id).sender(user_id!("@user_id:localhost"));
1001
1002        // Indexable dummy message required because RoomIndex is initialised
1003        // lazily.
1004        let dummy = f.text_msg("dummy").event_id(dummy_id);
1005
1006        let original = f.text_msg("This is a message").event_id(original_id);
1007
1008        let edit1 = f
1009            .text_msg("* A new message")
1010            .edit(original_id, RoomMessageEventContentWithoutRelation::text_plain("A new message"))
1011            .event_id(edit1_id);
1012
1013        let edit2 = f
1014            .text_msg("* An even newer message")
1015            .edit(
1016                original_id,
1017                RoomMessageEventContentWithoutRelation::text_plain("An even newer message"),
1018            )
1019            .event_id(edit2_id);
1020
1021        let edit3 = f
1022            .text_msg("* The newest message")
1023            .edit(
1024                original_id,
1025                RoomMessageEventContentWithoutRelation::text_plain("The newest message"),
1026            )
1027            .event_id(edit3_id);
1028
1029        server
1030            .sync_room(
1031                &client,
1032                JoinedRoomBuilder::new(room_id)
1033                    .add_timeline_event(dummy)
1034                    .add_timeline_event(edit1)
1035                    .add_timeline_event(edit2),
1036            )
1037            .await;
1038
1039        let results = room.search("message", 3, None).await.unwrap();
1040
1041        assert_eq!(results.events.len(), 0, "Search should return 0 results, got {results:?}");
1042
1043        // Adding the original after some pending edits should add the latest
1044        // edit instead of the original.
1045        server
1046            .sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(original))
1047            .await;
1048
1049        let results = room.search("message", 3, None).await.unwrap();
1050
1051        assert_eq!(results.events.len(), 1, "Search should return 1 result, got {results:?}");
1052        assert_eq!(
1053            results.events[0].1, edit2_id,
1054            "Search should return latest edit, got {:?}",
1055            results.events[0].1
1056        );
1057
1058        // Editing the original after it exists and there has been another edit
1059        // should delete the previous edits and add this one
1060        server.sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(edit3)).await;
1061
1062        let results = room.search("message", 3, None).await.unwrap();
1063
1064        assert_eq!(results.events.len(), 1, "Search should return 1 result, got {results:?}");
1065        assert_eq!(
1066            results.events[0].1, edit3_id,
1067            "Search should return latest edit, got {:?}",
1068            results.events[0].1
1069        );
1070    }
1071
1072    #[cfg(feature = "experimental-search")]
1073    #[async_test]
1074    async fn test_search_index_ignores_cross_sender_edit() {
1075        let room_id = room_id!("!room_id:localhost");
1076        let original_id = event_id!("$original");
1077        let edit_id = event_id!("$edit");
1078
1079        let server = MatrixMockServer::new().await;
1080        let client = server.client_builder().build().await;
1081
1082        let room = server.sync_joined_room(&client, room_id).await;
1083
1084        let f = EventFactory::new().room(room_id);
1085
1086        let original =
1087            f.text_msg("original alpha").sender(user_id!("@alice:localhost")).event_id(original_id);
1088
1089        // An edit from a different user than the original sender is not a valid
1090        // replacement and must be ignored.
1091        let malicious_edit = f
1092            .text_msg("* malicious beta")
1093            .edit(original_id, RoomMessageEventContentWithoutRelation::text_plain("malicious beta"))
1094            .sender(user_id!("@bob:localhost"))
1095            .event_id(edit_id);
1096
1097        server
1098            .sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(original))
1099            .await;
1100
1101        // The original message is indexed.
1102        let results = room.search("alpha", 3, None).await.unwrap();
1103        assert_eq!(results.events.len(), 1, "Original should be indexed, got {results:?}");
1104        assert_eq!(results.events[0].1, original_id, "unexpected event id: {results:?}");
1105
1106        server
1107            .sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(malicious_edit))
1108            .await;
1109
1110        // The forged edit's content must not be indexed.
1111        let results = room.search("beta", 3, None).await.unwrap();
1112        assert_eq!(results.events.len(), 0, "Cross-sender edit should be ignored, got {results:?}");
1113
1114        // The original message stays indexed.
1115        let results = room.search("alpha", 3, None).await.unwrap();
1116        assert_eq!(results.events.len(), 1, "Original should stay indexed, got {results:?}");
1117        assert_eq!(results.events[0].1, original_id, "unexpected event id: {results:?}");
1118    }
1119}
matrix_sdk/search_index/mod.rs

matrix_sdk/search_index/
mod.rs
