matrix_sdk/search_index/
mod.rs

1// Copyright 2025 The Matrix.org Foundation C.I.C.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The search index is an abstraction layer in the matrix-sdk for the
16//! matrix-sdk-search crate. It provides a [`SearchIndex`] which wraps
17//! multiple [`RoomIndex`].
18
19use std::{
20    collections::hash_map::HashMap,
21    num::NonZeroUsize,
22    path::{Path, PathBuf},
23    sync::Arc,
24    thread::available_parallelism,
25};
26
27use futures_util::{StreamExt as _, future::join_all, stream};
28use matrix_sdk_base::{
29    check_validity_of_replacement_events, deserialized_responses::TimelineEvent,
30};
31use matrix_sdk_search::{
32    error::IndexError,
33    index::{
34        IndexableEvent, RoomIndex, RoomIndexOperation, SearchResult, builder::RoomIndexBuilder,
35    },
36};
37use ruma::{
38    EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, RoomId,
39    events::{
40        AnySyncMessageLikeEvent, AnySyncTimelineEvent,
41        poll::{
42            start::SyncPollStartEvent,
43            unstable_start::{SyncUnstablePollStartEvent, UnstablePollStartEventContent},
44        },
45        room::{
46            message::{MessageType, OriginalSyncRoomMessageEvent, Relation, SyncRoomMessageEvent},
47            redaction::SyncRoomRedactionEvent,
48        },
49        sticker::SyncStickerEvent,
50    },
51    room_version_rules::RedactionRules,
52};
53use tokio::{
54    sync::{Mutex, MutexGuard},
55    task::spawn_blocking,
56};
57use tracing::{debug, warn};
58
59use crate::event_cache::RoomEventCache;
60
61type Key = String;
62
63/// Type of location to store [`RoomIndex`]
64#[derive(Clone, Debug)]
65pub enum SearchIndexStoreKind {
66    /// Store unencrypted in file system folder
67    UnencryptedDirectory(PathBuf),
68    /// Store encrypted, with the given high-entropy key, in file system folder
69    EncryptedDirectory(PathBuf, Key),
70    /// Store in memory
71    InMemory,
72}
73
74/// Object that handles inteeraction with [`RoomIndex`]'s for search
75#[derive(Clone, Debug)]
76pub struct SearchIndex {
77    /// HashMap that links each joined room to its RoomIndex
78    room_indexes: Arc<Mutex<HashMap<OwnedRoomId, RoomIndex>>>,
79
80    /// Base directory that stores the directories for each RoomIndex
81    search_index_store_kind: SearchIndexStoreKind,
82}
83
84impl SearchIndex {
85    /// Create a new [`SearchIndex`]
86    pub fn new(
87        room_indexes: Arc<Mutex<HashMap<OwnedRoomId, RoomIndex>>>,
88        search_index_store_kind: SearchIndexStoreKind,
89    ) -> Self {
90        Self { room_indexes, search_index_store_kind }
91    }
92
93    /// The directory the room indexes are stored in, if they are stored on
94    /// disk.
95    pub(crate) fn directory(&self) -> Option<&Path> {
96        match &self.search_index_store_kind {
97            SearchIndexStoreKind::UnencryptedDirectory(path)
98            | SearchIndexStoreKind::EncryptedDirectory(path, _) => Some(path),
99            SearchIndexStoreKind::InMemory => None,
100        }
101    }
102
103    /// The disk size of the room indexes, if they are stored on disk.
104    pub(crate) fn size(&self) -> Option<usize> {
105        fn dir_size(path: &Path) -> std::io::Result<u64> {
106            let mut size = 0;
107            for entry in std::fs::read_dir(path)? {
108                let entry = entry?;
109                let metadata = entry.metadata()?;
110                size += if metadata.is_dir() { dir_size(&entry.path())? } else { metadata.len() };
111            }
112            Ok(size)
113        }
114
115        let size = dir_size(self.directory()?).ok()?;
116        usize::try_from(size).ok()
117    }
118
119    /// Acquire [`SearchIndexGuard`] for this [`SearchIndex`].
120    pub async fn lock(&self) -> SearchIndexGuard<'_> {
121        SearchIndexGuard {
122            index_map: self.room_indexes.lock().await,
123            search_index_store_kind: &self.search_index_store_kind,
124        }
125    }
126
127    /// Open the indexes of the given rooms that exist on disk but aren't open
128    /// yet, concurrently and without holding the lock, instead of one by one
129    /// when they are first searched.
130    pub(crate) async fn open_indexes(&self, room_ids: impl Iterator<Item = OwnedRoomId>) {
131        let to_open: Vec<_> = {
132            let guard = self.lock().await;
133            room_ids
134                .filter(|room_id| {
135                    !guard.index_map.contains_key(room_id) && guard.has_index(room_id)
136                })
137                .collect()
138        };
139
140        let concurrency = available_parallelism().map_or(1, NonZeroUsize::get);
141
142        let mut opened = stream::iter(to_open)
143            .map(|room_id| {
144                let store_kind = self.search_index_store_kind.clone();
145                spawn_blocking(move || {
146                    let index = create_index(&store_kind, &room_id)
147                        .and_then(|index| index.warm_up().map(|()| index));
148                    (room_id, index)
149                })
150            })
151            .buffer_unordered(concurrency);
152
153        while let Some(task) = opened.next().await {
154            match task.expect("Task join error") {
155                // Keep the index opened meanwhile by the indexer, if any.
156                (room_id, Ok(index)) => {
157                    self.room_indexes.lock().await.entry(room_id).or_insert(index);
158                }
159                // The search will try again and report the error.
160                (room_id, Err(err)) => warn!(%room_id, "Failed to open the search index: {err}"),
161            }
162        }
163    }
164
165    /// Same as [`SearchIndexGuard::bulk_handle_timeline_event`], but only
166    /// locks the index once the events are parsed: parsing waits on the event
167    /// cache, and searches need the lock meanwhile.
168    pub(crate) async fn bulk_handle_timeline_event<T>(
169        &self,
170        events: T,
171        room_cache: &RoomEventCache,
172        room_id: &RoomId,
173        redaction_rules: &RedactionRules,
174    ) -> Result<(), IndexError>
175    where
176        T: Iterator<Item = TimelineEvent>,
177    {
178        let operations = parse_timeline_events(events, room_cache, redaction_rules).await;
179        self.lock().await.bulk_execute(operations, room_id)
180    }
181}
182
183/// Object that represents an acquired [`SearchIndex`].
184#[derive(Debug)]
185pub struct SearchIndexGuard<'a> {
186    /// Guard around the [`RoomIndex`] map
187    index_map: MutexGuard<'a, HashMap<OwnedRoomId, RoomIndex>>,
188
189    /// Base directory that stores the directories for each RoomIndex
190    search_index_store_kind: &'a SearchIndexStoreKind,
191}
192
193fn create_index(
194    search_index_store_kind: &SearchIndexStoreKind,
195    room_id: &RoomId,
196) -> Result<RoomIndex, IndexError> {
197    let index = match search_index_store_kind {
198        SearchIndexStoreKind::UnencryptedDirectory(path) => {
199            RoomIndexBuilder::new_on_disk(path.to_path_buf(), room_id).unencrypted().build()?
200        }
201        SearchIndexStoreKind::EncryptedDirectory(path, key) => {
202            RoomIndexBuilder::new_on_disk(path.to_path_buf(), room_id).encrypted(key).build()?
203        }
204        SearchIndexStoreKind::InMemory => RoomIndexBuilder::new_in_memory(room_id).build(),
205    };
206    Ok(index)
207}
208
209impl SearchIndexGuard<'_> {
210    /// Handle a [`RoomIndexOperation`] in the [`RoomIndex`] of a given
211    /// [`RoomId`]
212    ///
213    /// This which will add/remove/edit an event in the index based on the event
214    /// type.
215    ///
216    /// Prefer [`SearchIndexGuard::bulk_execute`] for multiple operations.
217    pub(crate) fn execute(
218        &mut self,
219        operation: RoomIndexOperation,
220        room_id: &RoomId,
221    ) -> Result<(), IndexError> {
222        if !self.index_map.contains_key(room_id) {
223            let index = create_index(self.search_index_store_kind, room_id)?;
224            self.index_map.insert(room_id.to_owned(), index);
225        }
226
227        let index = self.index_map.get_mut(room_id).expect("index should exist");
228
229        index.execute(operation)
230    }
231
232    /// Whether a [`RoomIndex`] exists for this room, in memory or on disk.
233    fn has_index(&self, room_id: &RoomId) -> bool {
234        self.index_map.contains_key(room_id)
235            || match self.search_index_store_kind {
236                SearchIndexStoreKind::UnencryptedDirectory(path)
237                | SearchIndexStoreKind::EncryptedDirectory(path, _) => {
238                    path.join(room_id.as_str()).exists()
239                }
240                SearchIndexStoreKind::InMemory => false,
241            }
242    }
243
244    /// Handle a [`RoomIndexOperation`] in the [`RoomIndex`] of a given
245    /// [`RoomId`]
246    ///
247    /// This which will add/remove/edit an event in the index based on the event
248    /// type.
249    pub(crate) fn bulk_execute(
250        &mut self,
251        operations: Vec<RoomIndexOperation>,
252        room_id: &RoomId,
253    ) -> Result<(), IndexError> {
254        // Don't create an index (which is costly) for nothing.
255        if operations.is_empty() {
256            return Ok(());
257        }
258
259        if !self.index_map.contains_key(room_id) {
260            let index = create_index(self.search_index_store_kind, room_id)?;
261            self.index_map.insert(room_id.to_owned(), index);
262        }
263
264        let index = self.index_map.get_mut(room_id).expect("index should exist");
265
266        index.bulk_execute(operations)
267    }
268
269    /// Search a [`Room`]'s index for the query and return at most
270    /// max_number_of_results results.
271    pub(crate) fn search(
272        &mut self,
273        query: &str,
274        max_number_of_results: usize,
275        pagination_offset: Option<usize>,
276        room_id: &RoomId,
277    ) -> Result<SearchResult, IndexError> {
278        // Nothing was ever indexed for this room: don't create an index (which
279        // is costly) only to find it empty.
280        if !self.has_index(room_id) {
281            return Ok(SearchResult { total_count: 0, events: Vec::new() });
282        }
283
284        if !self.index_map.contains_key(room_id) {
285            let index = create_index(self.search_index_store_kind, room_id)?;
286            self.index_map.insert(room_id.to_owned(), index);
287        }
288
289        let index = self.index_map.get_mut(room_id).expect("index should exist");
290
291        index.search(query, max_number_of_results, pagination_offset)
292    }
293
294    /// Given a [`TimelineEvent`] this function will derive a
295    /// [`RoomIndexOperation`], if it should be handled, and execute it;
296    /// returning the result.
297    ///
298    /// Prefer [`SearchIndexGuard::bulk_handle_timeline_event`] for multiple
299    /// events.
300    pub async fn handle_timeline_event(
301        &mut self,
302        event: TimelineEvent,
303        room_cache: &RoomEventCache,
304        room_id: &RoomId,
305        redaction_rules: &RedactionRules,
306    ) -> Result<(), IndexError> {
307        if let Some(index_operation) =
308            parse_timeline_event(room_cache, event, redaction_rules).await
309        {
310            self.execute(index_operation, room_id)
311        } else {
312            Ok(())
313        }
314    }
315
316    /// Run [`SearchIndexGuard::handle_timeline_event`] for multiple
317    /// [`TimelineEvent`].
318    pub async fn bulk_handle_timeline_event<T>(
319        &mut self,
320        events: T,
321        room_cache: &RoomEventCache,
322        room_id: &RoomId,
323        redaction_rules: &RedactionRules,
324    ) -> Result<(), IndexError>
325    where
326        T: Iterator<Item = TimelineEvent>,
327    {
328        let operations = parse_timeline_events(events, room_cache, redaction_rules).await;
329        self.bulk_execute(operations, room_id)
330    }
331}
332
333/// Derive the [`RoomIndexOperation`]s to execute for the given events.
334async fn parse_timeline_events(
335    events: impl Iterator<Item = TimelineEvent>,
336    room_cache: &RoomEventCache,
337    redaction_rules: &RedactionRules,
338) -> Vec<RoomIndexOperation> {
339    let futures = events.map(|ev| parse_timeline_event(room_cache, ev, redaction_rules));
340    join_all(futures).await.into_iter().flatten().collect()
341}
342
343/// Given an event id this function returns the most recent edit on said event
344/// or the event itself if there are no edits.
345async fn get_most_recent_edit(
346    cache: &RoomEventCache,
347    original: &EventId,
348) -> Option<OriginalSyncRoomMessageEvent> {
349    use ruma::events::{AnySyncTimelineEvent, relation::RelationType};
350
351    let Ok(Some((original_ev, related))) =
352        cache.find_event_with_relations(original, Some(vec![RelationType::Replacement])).await
353    else {
354        debug!("Couldn't find relations for {}", original);
355        return None;
356    };
357
358    // Only index valid replacements (matching sender, type, etc.); otherwise
359    // anyone could rewrite another user's indexed message. Fall back to the
360    // original event when there is no valid edit.
361    let latest = related
362        .iter()
363        .rev()
364        .find(|edit| {
365            check_validity_of_replacement_events(
366                original_ev.raw(),
367                original_ev.encryption_info().map(|info| &**info),
368                edit.raw(),
369                edit.encryption_info().map(|info| &**info),
370            )
371            .is_ok()
372        })
373        .unwrap_or(&original_ev);
374
375    match latest.raw().deserialize() {
376        Ok(AnySyncTimelineEvent::MessageLike(AnySyncMessageLikeEvent::RoomMessage(latest))) => {
377            latest.as_original().cloned()
378        }
379        _ => None,
380    }
381}
382
383/// Indexable text for a media message: its filename plus any caption.
384fn media_body(filename: &str, caption: Option<&str>) -> String {
385    match caption {
386        Some(caption) => format!("{filename} {caption}"),
387        None => filename.to_owned(),
388    }
389}
390
391/// Extract the searchable text from a room message, or `None` if its type
392/// carries no text to index.
393fn room_message_body(msgtype: &MessageType) -> Option<String> {
394    match msgtype {
395        MessageType::Text(content) => Some(content.body.clone()),
396        MessageType::Emote(content) => Some(content.body.clone()),
397        MessageType::Notice(content) => Some(content.body.clone()),
398        MessageType::ServerNotice(content) => Some(content.body.clone()),
399        MessageType::Location(content) => Some(content.body.clone()),
400        MessageType::Image(content) => Some(media_body(content.filename(), content.caption())),
401        MessageType::Video(content) => Some(media_body(content.filename(), content.caption())),
402        MessageType::Audio(content) => Some(media_body(content.filename(), content.caption())),
403        MessageType::File(content) => Some(media_body(content.filename(), content.caption())),
404        _ => None,
405    }
406}
407
408/// Build an [`IndexableEvent`] from a room message, or `None` if its type
409/// carries no searchable text.
410fn indexable_from_room_message(
411    event: &OriginalSyncRoomMessageEvent,
412    timestamp: Option<MilliSecondsSinceUnixEpoch>,
413) -> Option<IndexableEvent> {
414    let body = room_message_body(&event.content.msgtype)?;
415    let original_event_id = match &event.content.relates_to {
416        Some(Relation::Replacement(replacement)) => replacement.event_id.clone(),
417        _ => event.event_id.clone(),
418    };
419
420    Some(IndexableEvent::new(
421        event.event_id.clone(),
422        original_event_id,
423        event.sender.clone(),
424        timestamp,
425        body,
426    ))
427}
428
429/// If the given [`OriginalSyncRoomMessageEvent`] is an edit we make an
430/// [`RoomIndexOperation::Edit`] with the new most recent version of the
431/// original.
432async fn handle_possible_edit(
433    event: &OriginalSyncRoomMessageEvent,
434    timestamp: Option<MilliSecondsSinceUnixEpoch>,
435    cache: &RoomEventCache,
436) -> Option<RoomIndexOperation> {
437    if let Some(Relation::Replacement(replacement_data)) = &event.content.relates_to {
438        if let Some(recent) = get_most_recent_edit(cache, &replacement_data.event_id).await {
439            return Some(
440                indexable_from_room_message(&recent, timestamp).map_or(
441                    RoomIndexOperation::Noop,
442                    |indexable| {
443                        RoomIndexOperation::Edit(replacement_data.event_id.clone(), indexable)
444                    },
445                ),
446            );
447        } else {
448            return Some(RoomIndexOperation::Noop);
449        }
450    }
451    None
452}
453
454/// Return a [`RoomIndexOperation::Edit`] or [`RoomIndexOperation::Add`]
455/// depending on the message.
456async fn handle_room_message(
457    event: SyncRoomMessageEvent,
458    timestamp: Option<MilliSecondsSinceUnixEpoch>,
459    cache: &RoomEventCache,
460) -> Option<RoomIndexOperation> {
461    if let Some(event) = event.as_original() {
462        return handle_possible_edit(event, timestamp, cache).await.or(get_most_recent_edit(
463            cache,
464            &event.event_id,
465        )
466        .await
467        .and_then(|recent| {
468            indexable_from_room_message(&recent, timestamp).map(RoomIndexOperation::Add)
469        }));
470    }
471    None
472}
473
474/// Return a [`RoomIndexOperation`] removing a redacted event from the index, or
475/// re-adding the most recent remaining version if an edit was redacted.
476async fn handle_room_redaction(
477    event: SyncRoomRedactionEvent,
478    timestamp: Option<MilliSecondsSinceUnixEpoch>,
479    cache: &RoomEventCache,
480    rules: &RedactionRules,
481) -> Option<RoomIndexOperation> {
482    let redacted_event_id = event.redacts(rules)?;
483
484    // If the redacted event was a room message edit, re-add the most recent
485    // remaining version instead of just removing it.
486    if let Ok(Some(redacted_event)) = cache.find_event(redacted_event_id).await
487        && let Ok(AnySyncTimelineEvent::MessageLike(AnySyncMessageLikeEvent::RoomMessage(
488            redacted_event,
489        ))) = redacted_event.raw().deserialize()
490        && let Some(redacted_event) = redacted_event.as_original()
491        && let Some(operation) = handle_possible_edit(redacted_event, timestamp, cache).await
492    {
493        return Some(operation);
494    }
495
496    // Otherwise remove the redacted event from the index. This covers plain
497    // messages, stickers and polls.
498    Some(RoomIndexOperation::Remove(redacted_event_id.to_owned()))
499}
500
501/// Return a [`RoomIndexOperation::Add`] indexing a sticker's descriptive text.
502fn handle_sticker(
503    event: SyncStickerEvent,
504    timestamp: Option<MilliSecondsSinceUnixEpoch>,
505) -> Option<RoomIndexOperation> {
506    let event = event.as_original()?;
507
508    Some(RoomIndexOperation::Add(IndexableEvent::new(
509        event.event_id.clone(),
510        event.event_id.clone(),
511        event.sender.clone(),
512        timestamp,
513        event.content.body.clone(),
514    )))
515}
516
517/// Return a [`RoomIndexOperation::Add`] indexing an unstable poll's question
518/// and answers.
519///
520/// ponytail: only indexes the initial `New` poll — edits (`Replacement`) and
521/// poll ends are ignored. Add edit handling if editing a poll needs to update
522/// search results.
523fn handle_unstable_poll_start(
524    event: SyncUnstablePollStartEvent,
525    timestamp: Option<MilliSecondsSinceUnixEpoch>,
526) -> Option<RoomIndexOperation> {
527    let event = event.as_original()?;
528
529    let UnstablePollStartEventContent::New(content) = &event.content else {
530        return None;
531    };
532
533    let block = &content.poll_start;
534    let mut body = block.question.text.clone();
535    for answer in block.answers.iter() {
536        body.push(' ');
537        body.push_str(&answer.text);
538    }
539
540    Some(RoomIndexOperation::Add(IndexableEvent::new(
541        event.event_id.clone(),
542        event.event_id.clone(),
543        event.sender.clone(),
544        timestamp,
545        body,
546    )))
547}
548
549/// Return a [`RoomIndexOperation::Add`] indexing a stable poll's question and
550/// answers.
551///
552/// ponytail: like [`handle_unstable_poll_start`], edits and poll ends are
553/// ignored — only the initial poll is indexed.
554fn handle_poll_start(
555    event: SyncPollStartEvent,
556    timestamp: Option<MilliSecondsSinceUnixEpoch>,
557) -> Option<RoomIndexOperation> {
558    let event = event.as_original()?;
559
560    // Skip poll edits, matching the unstable poll handling.
561    if let Some(Relation::Replacement(_)) = &event.content.relates_to {
562        return None;
563    }
564
565    let block = &event.content.poll;
566    let mut body = block.question.text.find_plain()?.to_owned();
567    for answer in block.answers.iter() {
568        if let Some(text) = answer.text.find_plain() {
569            body.push(' ');
570            body.push_str(text);
571        }
572    }
573
574    Some(RoomIndexOperation::Add(IndexableEvent::new(
575        event.event_id.clone(),
576        event.event_id.clone(),
577        event.sender.clone(),
578        timestamp,
579        body,
580    )))
581}
582
583/// Prepare a [`TimelineEvent`] into a [`RoomIndexOperation`] for search
584/// indexing.
585async fn parse_timeline_event(
586    cache: &RoomEventCache,
587    event: TimelineEvent,
588    redaction_rules: &RedactionRules,
589) -> Option<RoomIndexOperation> {
590    use ruma::events::AnySyncTimelineEvent;
591
592    if event.kind.is_utd() {
593        return None;
594    }
595
596    let timestamp = event.timestamp();
597
598    match event.raw().deserialize() {
599        Ok(event) => match event {
600            AnySyncTimelineEvent::MessageLike(event) => match event {
601                AnySyncMessageLikeEvent::RoomMessage(event) => {
602                    handle_room_message(event, timestamp, cache).await
603                }
604                AnySyncMessageLikeEvent::RoomRedaction(event) => {
605                    handle_room_redaction(event, timestamp, cache, redaction_rules).await
606                }
607                AnySyncMessageLikeEvent::Sticker(event) => handle_sticker(event, timestamp),
608                AnySyncMessageLikeEvent::PollStart(event) => handle_poll_start(event, timestamp),
609                AnySyncMessageLikeEvent::UnstablePollStart(event) => {
610                    handle_unstable_poll_start(event, timestamp)
611                }
612                _ => None,
613            },
614            AnySyncTimelineEvent::State(_) => None,
615        },
616
617        Err(e) => {
618            warn!("failed to parse event: {e:?}");
619            None
620        }
621    }
622}
623
624#[cfg(test)]
625mod tests {
626    use matrix_sdk_test::{JoinedRoomBuilder, async_test, event_factory::EventFactory};
627    use ruma::{
628        event_id, events::room::message::RoomMessageEventContentWithoutRelation, room_id, user_id,
629    };
630
631    use crate::test_utils::mocks::MatrixMockServer;
632
633    #[cfg(feature = "experimental-search")]
634    #[async_test]
635    async fn test_search_index_size() {
636        use super::SearchIndexStoreKind;
637
638        let dir = tempfile::tempdir().unwrap();
639        let mock_server = MatrixMockServer::new().await;
640        let client = mock_server
641            .client_builder()
642            .on_builder(|builder| {
643                builder.search_index_store(SearchIndexStoreKind::UnencryptedDirectory(
644                    dir.path().join("search"),
645                ))
646            })
647            .build()
648            .await;
649
650        // Nothing indexed yet: the directory doesn't even exist.
651        assert_eq!(client.get_store_sizes().await.unwrap().search_index, None);
652
653        let room_id = room_id!("!room_id:localhost");
654        mock_server
655            .sync_room(
656                &client,
657                JoinedRoomBuilder::new(room_id).add_timeline_event(
658                    EventFactory::new()
659                        .text_msg("this is a sentence")
660                        .room(room_id)
661                        .sender(user_id!("@user_id:localhost")),
662                ),
663            )
664            .await;
665
666        // Let the indexer process the event.
667        for _ in 0..50 {
668            if client.get_store_sizes().await.unwrap().search_index.is_some_and(|size| size > 0) {
669                return;
670            }
671            crate::sleep::sleep(std::time::Duration::from_millis(100)).await;
672        }
673        panic!("the search index size was never reported");
674    }
675
676    #[cfg(feature = "experimental-search")]
677    #[async_test]
678    async fn test_no_index_is_created_for_nothing() {
679        use super::{SearchIndex, SearchIndexStoreKind};
680
681        let dir = tempfile::tempdir().unwrap();
682        let search_index = SearchIndex::new(
683            Default::default(),
684            SearchIndexStoreKind::UnencryptedDirectory(dir.path().to_path_buf()),
685        );
686        let room_id = room_id!("!room_id:localhost");
687        let mut guard = search_index.lock().await;
688
689        // Searching a room that was never indexed finds nothing…
690        let result = guard.search("sentence", 10, None, room_id).unwrap();
691        assert_eq!(result.total_count, 0);
692        assert!(result.events.is_empty());
693
694        // … and so does an empty batch of operations: none creates an index.
695        guard.bulk_execute(Vec::new(), room_id).unwrap();
696
697        assert!(!dir.path().join(room_id.as_str()).exists());
698    }
699
700    #[cfg(feature = "experimental-search")]
701    #[async_test]
702    async fn test_sync_message_is_indexed() {
703        let mock_server = MatrixMockServer::new().await;
704        let client = mock_server.client_builder().build().await;
705
706        let room_id = room_id!("!room_id:localhost");
707        let event_id = event_id!("$event_id:localost");
708        let user_id = user_id!("@user_id:localost");
709
710        let event_factory = EventFactory::new();
711        let room = mock_server
712            .sync_room(
713                &client,
714                JoinedRoomBuilder::new(room_id).add_timeline_bulk(vec![
715                    event_factory
716                        .text_msg("this is a sentence")
717                        .event_id(event_id)
718                        .sender(user_id)
719                        .into_raw_sync(),
720                ]),
721            )
722            .await;
723
724        let response = room.search("this", 5, None).await.expect("search should have 1 result");
725
726        assert_eq!(response.events.len(), 1, "unexpected numbers of responses: {response:?}");
727        assert_eq!(response.events[0].1, event_id, "event id doesn't match: {response:?}");
728    }
729
730    #[cfg(feature = "experimental-search")]
731    #[async_test]
732    async fn test_sync_media_message_is_indexed() {
733        use ruma::owned_mxc_uri;
734
735        let mock_server = MatrixMockServer::new().await;
736        let client = mock_server.client_builder().build().await;
737
738        let room_id = room_id!("!room_id:localhost");
739        let image_id = event_id!("$image_id:localhost");
740        let file_id = event_id!("$file_id:localhost");
741        let user_id = user_id!("@user_id:localhost");
742
743        let f = EventFactory::new();
744        let room = mock_server
745            .sync_room(
746                &client,
747                JoinedRoomBuilder::new(room_id).add_timeline_bulk(vec![
748                    f.image("holiday_beach.jpg".to_owned(), owned_mxc_uri!("mxc://localhost/1"))
749                        .caption(Some("sunset over the ocean".to_owned()), None)
750                        .event_id(image_id)
751                        .sender(user_id)
752                        .into_raw_sync(),
753                    f.image("quarterly_report.pdf".to_owned(), owned_mxc_uri!("mxc://localhost/2"))
754                        .event_id(file_id)
755                        .sender(user_id)
756                        .into_raw_sync(),
757                ]),
758            )
759            .await;
760
761        // The caption is indexed.
762        let response = room.search("sunset", 5, None).await.unwrap();
763        assert_eq!(response.events.len(), 1, "unexpected results for caption search: {response:?}");
764        assert_eq!(response.events[0].1, image_id, "event id doesn't match: {response:?}");
765
766        // The filename is indexed.
767        let response = room.search("holiday_beach", 5, None).await.unwrap();
768        assert_eq!(
769            response.events.len(),
770            1,
771            "unexpected results for filename search: {response:?}"
772        );
773        assert_eq!(response.events[0].1, image_id, "event id doesn't match: {response:?}");
774
775        // A media message without a caption still indexes its filename.
776        let response = room.search("quarterly_report", 5, None).await.unwrap();
777        assert_eq!(
778            response.events.len(),
779            1,
780            "unexpected results for filename search: {response:?}"
781        );
782        assert_eq!(response.events[0].1, file_id, "event id doesn't match: {response:?}");
783    }
784
785    #[cfg(feature = "experimental-search")]
786    #[async_test]
787    async fn test_sync_sticker_and_poll_are_indexed() {
788        use ruma::{events::room::ImageInfo, owned_mxc_uri};
789
790        let mock_server = MatrixMockServer::new().await;
791        let client = mock_server.client_builder().build().await;
792
793        let room_id = room_id!("!room_id:localhost");
794        let sticker_id = event_id!("$sticker_id:localhost");
795        let poll_id = event_id!("$poll_id:localhost");
796        let user_id = user_id!("@user_id:localhost");
797
798        let f = EventFactory::new().room(room_id).sender(user_id);
799        let room = mock_server
800            .sync_room(
801                &client,
802                JoinedRoomBuilder::new(room_id).add_timeline_bulk(vec![
803                    f.sticker(
804                        "a waving cat",
805                        ImageInfo::new(),
806                        owned_mxc_uri!("mxc://localhost/1"),
807                    )
808                    .event_id(sticker_id)
809                    .into_raw_sync(),
810                    f.poll_start("fallback", "favourite cheese?", vec!["comté", "gruyère"])
811                        .event_id(poll_id)
812                        .into_raw_sync(),
813                ]),
814            )
815            .await;
816
817        // The sticker's description is indexed.
818        let response = room.search("waving", 5, None).await.unwrap();
819        assert_eq!(response.events.len(), 1, "unexpected results for sticker search: {response:?}");
820        assert_eq!(response.events[0].1, sticker_id, "event id doesn't match: {response:?}");
821
822        // The poll question is indexed.
823        let response = room.search("cheese", 5, None).await.unwrap();
824        assert_eq!(
825            response.events.len(),
826            1,
827            "unexpected results for poll question search: {response:?}"
828        );
829        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
830
831        // The poll answers are indexed.
832        let response = room.search("gruyère", 5, None).await.unwrap();
833        assert_eq!(
834            response.events.len(),
835            1,
836            "unexpected results for poll answer search: {response:?}"
837        );
838        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
839    }
840
841    #[cfg(feature = "experimental-search")]
842    #[async_test]
843    async fn test_sync_stable_poll_is_indexed() {
844        use ruma::events::{
845            message::TextContentBlock,
846            poll::start::{PollAnswer, PollAnswers, PollContentBlock, PollStartEventContent},
847        };
848
849        let mock_server = MatrixMockServer::new().await;
850        let client = mock_server.client_builder().build().await;
851
852        let room_id = room_id!("!room_id:localhost");
853        let poll_id = event_id!("$stable_poll_id:localhost");
854        let user_id = user_id!("@user_id:localhost");
855
856        let answers: PollAnswers = vec![
857            PollAnswer::new("0".to_owned(), TextContentBlock::plain("comté")),
858            PollAnswer::new("1".to_owned(), TextContentBlock::plain("gruyère")),
859        ]
860        .try_into()
861        .unwrap();
862        let poll = PollContentBlock::new(TextContentBlock::plain("favourite cheese?"), answers);
863        let content = PollStartEventContent::new(TextContentBlock::plain("fallback"), poll);
864
865        let f = EventFactory::new().room(room_id).sender(user_id);
866        let room = mock_server
867            .sync_room(
868                &client,
869                JoinedRoomBuilder::new(room_id)
870                    .add_timeline_bulk(vec![f.event(content).event_id(poll_id).into_raw_sync()]),
871            )
872            .await;
873
874        // The poll question is indexed.
875        let response = room.search("cheese", 5, None).await.unwrap();
876        assert_eq!(
877            response.events.len(),
878            1,
879            "unexpected results for poll question search: {response:?}"
880        );
881        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
882
883        // The poll answers are indexed.
884        let response = room.search("gruyère", 5, None).await.unwrap();
885        assert_eq!(
886            response.events.len(),
887            1,
888            "unexpected results for poll answer search: {response:?}"
889        );
890        assert_eq!(response.events[0].1, poll_id, "event id doesn't match: {response:?}");
891    }
892
893    #[cfg(feature = "experimental-search")]
894    #[async_test]
895    async fn test_search_index_edit_ordering() {
896        let room_id = room_id!("!room_id:localhost");
897        let dummy_id = event_id!("$dummy");
898        let edit1_id = event_id!("$edit1");
899        let edit2_id = event_id!("$edit2");
900        let edit3_id = event_id!("$edit3");
901        let original_id = event_id!("$original");
902
903        let server = MatrixMockServer::new().await;
904        let client = server.client_builder().build().await;
905
906        let room = server.sync_joined_room(&client, room_id).await;
907
908        let f = EventFactory::new().room(room_id).sender(user_id!("@user_id:localhost"));
909
910        // Indexable dummy message required because RoomIndex is initialised
911        // lazily.
912        let dummy = f.text_msg("dummy").event_id(dummy_id);
913
914        let original = f.text_msg("This is a message").event_id(original_id);
915
916        let edit1 = f
917            .text_msg("* A new message")
918            .edit(original_id, RoomMessageEventContentWithoutRelation::text_plain("A new message"))
919            .event_id(edit1_id);
920
921        let edit2 = f
922            .text_msg("* An even newer message")
923            .edit(
924                original_id,
925                RoomMessageEventContentWithoutRelation::text_plain("An even newer message"),
926            )
927            .event_id(edit2_id);
928
929        let edit3 = f
930            .text_msg("* The newest message")
931            .edit(
932                original_id,
933                RoomMessageEventContentWithoutRelation::text_plain("The newest message"),
934            )
935            .event_id(edit3_id);
936
937        server
938            .sync_room(
939                &client,
940                JoinedRoomBuilder::new(room_id)
941                    .add_timeline_event(dummy)
942                    .add_timeline_event(edit1)
943                    .add_timeline_event(edit2),
944            )
945            .await;
946
947        let results = room.search("message", 3, None).await.unwrap();
948
949        assert_eq!(results.events.len(), 0, "Search should return 0 results, got {results:?}");
950
951        // Adding the original after some pending edits should add the latest
952        // edit instead of the original.
953        server
954            .sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(original))
955            .await;
956
957        let results = room.search("message", 3, None).await.unwrap();
958
959        assert_eq!(results.events.len(), 1, "Search should return 1 result, got {results:?}");
960        assert_eq!(
961            results.events[0].1, edit2_id,
962            "Search should return latest edit, got {:?}",
963            results.events[0].1
964        );
965
966        // Editing the original after it exists and there has been another edit
967        // should delete the previous edits and add this one
968        server.sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(edit3)).await;
969
970        let results = room.search("message", 3, None).await.unwrap();
971
972        assert_eq!(results.events.len(), 1, "Search should return 1 result, got {results:?}");
973        assert_eq!(
974            results.events[0].1, edit3_id,
975            "Search should return latest edit, got {:?}",
976            results.events[0].1
977        );
978    }
979
980    #[cfg(feature = "experimental-search")]
981    #[async_test]
982    async fn test_search_index_ignores_cross_sender_edit() {
983        let room_id = room_id!("!room_id:localhost");
984        let original_id = event_id!("$original");
985        let edit_id = event_id!("$edit");
986
987        let server = MatrixMockServer::new().await;
988        let client = server.client_builder().build().await;
989
990        let room = server.sync_joined_room(&client, room_id).await;
991
992        let f = EventFactory::new().room(room_id);
993
994        let original =
995            f.text_msg("original alpha").sender(user_id!("@alice:localhost")).event_id(original_id);
996
997        // An edit from a different user than the original sender is not a valid
998        // replacement and must be ignored.
999        let malicious_edit = f
1000            .text_msg("* malicious beta")
1001            .edit(original_id, RoomMessageEventContentWithoutRelation::text_plain("malicious beta"))
1002            .sender(user_id!("@bob:localhost"))
1003            .event_id(edit_id);
1004
1005        server
1006            .sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(original))
1007            .await;
1008
1009        // The original message is indexed.
1010        let results = room.search("alpha", 3, None).await.unwrap();
1011        assert_eq!(results.events.len(), 1, "Original should be indexed, got {results:?}");
1012        assert_eq!(results.events[0].1, original_id, "unexpected event id: {results:?}");
1013
1014        server
1015            .sync_room(&client, JoinedRoomBuilder::new(room_id).add_timeline_event(malicious_edit))
1016            .await;
1017
1018        // The forged edit's content must not be indexed.
1019        let results = room.search("beta", 3, None).await.unwrap();
1020        assert_eq!(results.events.len(), 0, "Cross-sender edit should be ignored, got {results:?}");
1021
1022        // The original message stays indexed.
1023        let results = room.search("alpha", 3, None).await.unwrap();
1024        assert_eq!(results.events.len(), 1, "Original should stay indexed, got {results:?}");
1025        assert_eq!(results.events[0].1, original_id, "unexpected event id: {results:?}");
1026    }
1027}
matrix_sdk/search_index/mod.rs

matrix_sdk/search_index/
mod.rs
