diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py
index 6fde273ce5..b5ffd8e67d 100644
--- a/synapse/handlers/sliding_sync.py
+++ b/synapse/handlers/sliding_sync.py
@@ -35,6 +35,7 @@ from synapse.api.constants import AccountDataTypes, EventTypes, Membership
 from synapse.events import EventBase
 from synapse.rest.client.models import SlidingSyncBody
 from synapse.types import JsonMapping, Requester, RoomStreamToken, StreamToken, UserID
+from synapse.types.state import StateFilter
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -340,7 +341,7 @@ class SlidingSyncHandler:
                     # from/to tokens but some of the streams don't support looking back
                     # in time (like global account_data).
                     filtered_room_ids = await self.filter_rooms(
-                        sync_config.user, room_id_set, list_config.filters
+                        sync_config.user, room_id_set, list_config.filters, to_token
                     )
                 # TODO: Apply sorts
                 sorted_room_ids = sorted(filtered_room_ids)
@@ -619,9 +620,15 @@ class SlidingSyncHandler:
         user: UserID,
         room_id_set: AbstractSet[str],
         filters: SlidingSyncConfig.SlidingSyncList.Filters,
+        to_token: StreamToken,
     ) -> AbstractSet[str]:
         """
         Filter rooms based on the sync request.
+
+        Args:
+            user:
+            room_id_set: The set of room IDs to filter down
+            filters: The filters to apply
         """
         user_id = user.to_string()
 
@@ -664,15 +671,18 @@ class SlidingSyncHandler:
 
         # Filter for encrypted rooms
         if filters.is_encrypted is not None:
-            # Make a copy so we don't run into an error: `Set changed size during iteration`
+            # Make a copy so we don't run into an error: `Set changed size during
+            # iteration`, when we filter out and remove items
             for room_id in list(filtered_room_id_set):
                 # TODO: Is there a good method to look up all rooms at once? (N+1 query problem)
-                is_encrypted = (
-                    # TODO: Get state at the `to_token` instead of the current state
-                    await self.storage_controllers.state.get_current_state_event(
-                        room_id, EventTypes.RoomEncryption, ""
-                    )
+                state_at_to_token = await self.storage_controllers.state.get_state_at(
+                    room_id,
+                    to_token,
+                    state_filter=StateFilter.from_types(
+                        [(EventTypes.RoomEncryption, "")]
+                    ),
                 )
+                is_encrypted = state_at_to_token.get((EventTypes.RoomEncryption, ""))
 
                 # If we're looking for encrypted rooms, filter out rooms that are not
                 # encrypted and vice versa
diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py
index 599e3dbc10..0c0f229f2e 100644
--- a/tests/handlers/test_sliding_sync.py
+++ b/tests/handlers/test_sliding_sync.py
@@ -1140,6 +1140,7 @@ class FilterRoomsTestCase(HomeserverTestCase):
     def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
         self.sliding_sync_handler = self.hs.get_sliding_sync_handler()
         self.store = self.hs.get_datastores().main
+        self.event_sources = hs.get_event_sources()
 
     def _create_dm_room(
         self,
@@ -1213,6 +1214,8 @@ class FilterRoomsTestCase(HomeserverTestCase):
             invitee_tok=user2_tok,
         )
 
+        after_rooms_token = self.event_sources.get_current_token()
+
         # TODO: Better way to avoid the circular import? (see
         # https://github.com/element-hq/synapse/pull/17187#discussion_r1619492779)
         from synapse.handlers.sliding_sync import SlidingSyncConfig
@@ -1226,7 +1229,10 @@ class FilterRoomsTestCase(HomeserverTestCase):
         # Filter the rooms
         truthy_filtered_room_ids = self.get_success(
             self.sliding_sync_handler.filter_rooms(
-                UserID.from_string(user1_id), {room_id, dm_room_id}, truthy_filters
+                UserID.from_string(user1_id),
+                {room_id, dm_room_id},
+                truthy_filters,
+                after_rooms_token,
             )
         )
 
@@ -1241,7 +1247,10 @@ class FilterRoomsTestCase(HomeserverTestCase):
         # Filter the rooms
         falsy_filtered_room_ids = self.get_success(
             self.sliding_sync_handler.filter_rooms(
-                UserID.from_string(user1_id), {room_id, dm_room_id}, falsy_filters
+                UserID.from_string(user1_id),
+                {room_id, dm_room_id},
+                falsy_filters,
+                after_rooms_token,
             )
         )
 
@@ -1274,39 +1283,35 @@ class FilterRoomsTestCase(HomeserverTestCase):
             tok=user1_tok,
         )
 
+        after_rooms_token = self.event_sources.get_current_token()
+
         # TODO: Better way to avoid the circular import? (see
         # https://github.com/element-hq/synapse/pull/17187#discussion_r1619492779)
         from synapse.handlers.sliding_sync import SlidingSyncConfig
 
         # Try with `is_encrypted=True`
-        # -----------------------------
-        truthy_filters = SlidingSyncConfig.SlidingSyncList.Filters(
-            is_encrypted=True,
-        )
-
-        # Filter the rooms
         truthy_filtered_room_ids = self.get_success(
             self.sliding_sync_handler.filter_rooms(
                 UserID.from_string(user1_id),
                 {room_id, encrypted_room_id},
-                truthy_filters,
+                SlidingSyncConfig.SlidingSyncList.Filters(
+                    is_encrypted=True,
+                ),
+                after_rooms_token,
             )
         )
 
         self.assertEqual(truthy_filtered_room_ids, {encrypted_room_id})
 
         # Try with `is_encrypted=False`
-        # -----------------------------
-        falsy_filters = SlidingSyncConfig.SlidingSyncList.Filters(
-            is_encrypted=False,
-        )
-
-        # Filter the rooms
         falsy_filtered_room_ids = self.get_success(
             self.sliding_sync_handler.filter_rooms(
                 UserID.from_string(user1_id),
                 {room_id, encrypted_room_id},
-                falsy_filters,
+                SlidingSyncConfig.SlidingSyncList.Filters(
+                    is_encrypted=False,
+                ),
+                after_rooms_token,
             )
         )