x86_64/structures/paging/mapper/
recursive_page_table.rs

1//! Access the page tables through a recursively mapped level 4 table.
2
3use core::fmt;
4
5use super::*;
6use crate::registers::control::Cr3;
7use crate::structures::paging::page_table::PageTableLevel;
8use crate::structures::paging::{
9    page::{AddressNotAligned, NotGiantPageSize},
10    page_table::{FrameError, PageTable, PageTableEntry},
11    PageTableIndex,
12};
13
14/// A recursive page table is a last level page table with an entry mapped to the table itself.
15///
16/// This recursive mapping allows accessing all page tables in the hierarchy:
17///
18/// - To access the level 4 page table, we “loop“ (i.e. follow the recursively mapped entry) four
19///   times.
20/// - To access a level 3 page table, we “loop” three times and then use the level 4 index.
21/// - To access a level 2 page table, we “loop” two times, then use the level 4 index, then the
22///   level 3 index.
23/// - To access a level 1 page table, we “loop” once, then use the level 4 index, then the
24///   level 3 index, then the level 2 index.
25///
26/// This struct implements the `Mapper` trait.
27///
28/// The page table flags `PRESENT` and `WRITABLE` are always set for higher level page table
29/// entries, even if not specified, because the design of the recursive page table requires it.
30#[derive(Debug)]
31pub struct RecursivePageTable<'a> {
32    p4: &'a mut PageTable,
33    recursive_index: PageTableIndex,
34}
35
36impl<'a> RecursivePageTable<'a> {
37    /// Creates a new RecursivePageTable from the passed level 4 PageTable.
38    ///
39    /// The page table must be recursively mapped, that means:
40    ///
41    /// - The page table must have one recursive entry, i.e. an entry that points to the table
42    ///   itself.
43    ///     - The reference must use that “loop”, i.e. be of the form `0o_xxx_xxx_xxx_xxx_0000`
44    ///       where `xxx` is the recursive entry.
45    /// - The page table must be active, i.e. the CR3 register must contain its physical address.
46    ///
47    /// Otherwise `Err(())` is returned.
48    ///
49    /// ## Safety
50    ///
51    /// Note that creating a `PageTable` with recursive index 511 is unsound
52    /// because allocating the last byte of the address space can lead to pointer
53    /// overflows and undefined behavior. For more details, see the discussions
54    /// [on Zulip](https://rust-lang.zulipchat.com/#narrow/stream/136281-t-opsem/topic/end-of-address-space)
55    /// and [in the `unsafe-code-guidelines ` repo](https://github.com/rust-lang/unsafe-code-guidelines/issues/420).
56    #[inline]
57    pub fn new(table: &'a mut PageTable) -> Result<Self, InvalidPageTable> {
58        let page = Page::containing_address(VirtAddr::new(table as *const _ as u64));
59        let recursive_index = page.p4_index();
60
61        if page.p3_index() != recursive_index
62            || page.p2_index() != recursive_index
63            || page.p1_index() != recursive_index
64        {
65            return Err(InvalidPageTable::NotRecursive);
66        }
67        if Ok(Cr3::read().0) != table[recursive_index].frame() {
68            return Err(InvalidPageTable::NotActive);
69        }
70
71        Ok(RecursivePageTable {
72            p4: table,
73            recursive_index,
74        })
75    }
76
77    /// Creates a new RecursivePageTable without performing any checks.
78    ///
79    /// ## Safety
80    ///
81    /// The given page table must be a level 4 page table that is active in the
82    /// CPU (i.e. loaded in the CR3 register). The `recursive_index` parameter
83    /// must be the index of the recursively mapped entry of that page table.
84    #[inline]
85    pub unsafe fn new_unchecked(table: &'a mut PageTable, recursive_index: PageTableIndex) -> Self {
86        RecursivePageTable {
87            p4: table,
88            recursive_index,
89        }
90    }
91
92    /// Returns an immutable reference to the wrapped level 4 `PageTable` instance.
93    pub fn level_4_table(&self) -> &PageTable {
94        self.p4
95    }
96
97    /// Returns a mutable reference to the wrapped level 4 `PageTable` instance.
98    pub fn level_4_table_mut(&mut self) -> &mut PageTable {
99        self.p4
100    }
101
102    /// Internal helper function to create the page table of the next level if needed.
103    ///
104    /// If the passed entry is unused, a new frame is allocated from the given allocator, zeroed,
105    /// and the entry is updated to that address. If the passed entry is already mapped, the next
106    /// table is returned directly.
107    ///
108    /// The page table flags `PRESENT` and `WRITABLE` are always set for higher level page table
109    /// entries, even if not specified in the `insert_flags`, because the design of the
110    /// recursive page table requires it.
111    ///
112    /// The `next_page_table` page must be the page of the next page table in the hierarchy.
113    ///
114    /// Returns `MapToError::FrameAllocationFailed` if the entry is unused and the allocator
115    /// returned `None`. Returns `MapToError::ParentEntryHugePage` if the `HUGE_PAGE` flag is set
116    /// in the passed entry.
117    unsafe fn create_next_table<'b, A, S: PageSize>(
118        entry: &'b mut PageTableEntry,
119        next_table_page: Page,
120        insert_flags: PageTableFlags,
121        allocator: &mut A,
122    ) -> Result<&'b mut PageTable, MapToError<S>>
123    where
124        A: FrameAllocator<Size4KiB> + ?Sized,
125    {
126        /// This inner function is used to limit the scope of `unsafe`.
127        ///
128        /// This is a safe function, so we need to use `unsafe` blocks when we do something unsafe.
129        fn inner<'b, A, S: PageSize>(
130            entry: &'b mut PageTableEntry,
131            next_table_page: Page,
132            insert_flags: PageTableFlags,
133            allocator: &mut A,
134        ) -> Result<&'b mut PageTable, MapToError<S>>
135        where
136            A: FrameAllocator<Size4KiB> + ?Sized,
137        {
138            use crate::structures::paging::PageTableFlags as Flags;
139
140            let created;
141
142            if entry.is_unused() {
143                if let Some(frame) = allocator.allocate_frame() {
144                    entry.set_frame(frame, Flags::PRESENT | Flags::WRITABLE | insert_flags);
145                    created = true;
146                } else {
147                    return Err(MapToError::FrameAllocationFailed);
148                }
149            } else {
150                if !insert_flags.is_empty() && !entry.flags().contains(insert_flags) {
151                    entry.set_flags(entry.flags() | insert_flags);
152                }
153                created = false;
154            }
155            if entry.flags().contains(Flags::HUGE_PAGE) {
156                return Err(MapToError::ParentEntryHugePage);
157            }
158
159            let page_table_ptr = next_table_page.start_address().as_mut_ptr();
160            let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) };
161            if created {
162                page_table.zero();
163            }
164            Ok(page_table)
165        }
166
167        inner(entry, next_table_page, insert_flags, allocator)
168    }
169
170    /// Helper function for implementing Mapper. Safe to limit the scope of unsafe, see
171    /// https://github.com/rust-lang/rfcs/pull/2585.
172    fn map_to_1gib<A>(
173        &mut self,
174        page: Page<Size1GiB>,
175        frame: PhysFrame<Size1GiB>,
176        flags: PageTableFlags,
177        parent_table_flags: PageTableFlags,
178        allocator: &mut A,
179    ) -> Result<MapperFlush<Size1GiB>, MapToError<Size1GiB>>
180    where
181        A: FrameAllocator<Size4KiB> + ?Sized,
182    {
183        use crate::structures::paging::PageTableFlags as Flags;
184        let p4 = &mut self.p4;
185
186        let p3_page = p3_page(page, self.recursive_index);
187        let p3 = unsafe {
188            Self::create_next_table(
189                &mut p4[page.p4_index()],
190                p3_page,
191                parent_table_flags,
192                allocator,
193            )?
194        };
195
196        if !p3[page.p3_index()].is_unused() {
197            return Err(MapToError::PageAlreadyMapped(frame));
198        }
199        p3[page.p3_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE);
200
201        Ok(MapperFlush::new(page))
202    }
203
204    /// Helper function for implementing Mapper. Safe to limit the scope of unsafe, see
205    /// https://github.com/rust-lang/rfcs/pull/2585.
206    fn map_to_2mib<A>(
207        &mut self,
208        page: Page<Size2MiB>,
209        frame: PhysFrame<Size2MiB>,
210        flags: PageTableFlags,
211        parent_table_flags: PageTableFlags,
212        allocator: &mut A,
213    ) -> Result<MapperFlush<Size2MiB>, MapToError<Size2MiB>>
214    where
215        A: FrameAllocator<Size4KiB> + ?Sized,
216    {
217        use crate::structures::paging::PageTableFlags as Flags;
218        let p4 = &mut self.p4;
219
220        let p3_page = p3_page(page, self.recursive_index);
221        let p3 = unsafe {
222            Self::create_next_table(
223                &mut p4[page.p4_index()],
224                p3_page,
225                parent_table_flags,
226                allocator,
227            )?
228        };
229
230        let p2_page = p2_page(page, self.recursive_index);
231        let p2 = unsafe {
232            Self::create_next_table(
233                &mut p3[page.p3_index()],
234                p2_page,
235                parent_table_flags,
236                allocator,
237            )?
238        };
239
240        if !p2[page.p2_index()].is_unused() {
241            return Err(MapToError::PageAlreadyMapped(frame));
242        }
243        p2[page.p2_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE);
244
245        Ok(MapperFlush::new(page))
246    }
247
248    /// Helper function for implementing Mapper. Safe to limit the scope of unsafe, see
249    /// https://github.com/rust-lang/rfcs/pull/2585.
250    fn map_to_4kib<A>(
251        &mut self,
252        page: Page<Size4KiB>,
253        frame: PhysFrame<Size4KiB>,
254        flags: PageTableFlags,
255        parent_table_flags: PageTableFlags,
256        allocator: &mut A,
257    ) -> Result<MapperFlush<Size4KiB>, MapToError<Size4KiB>>
258    where
259        A: FrameAllocator<Size4KiB> + ?Sized,
260    {
261        let p4 = &mut self.p4;
262
263        let p3_page = p3_page(page, self.recursive_index);
264        let p3 = unsafe {
265            Self::create_next_table(
266                &mut p4[page.p4_index()],
267                p3_page,
268                parent_table_flags,
269                allocator,
270            )?
271        };
272
273        let p2_page = p2_page(page, self.recursive_index);
274        let p2 = unsafe {
275            Self::create_next_table(
276                &mut p3[page.p3_index()],
277                p2_page,
278                parent_table_flags,
279                allocator,
280            )?
281        };
282
283        let p1_page = p1_page(page, self.recursive_index);
284        let p1 = unsafe {
285            Self::create_next_table(
286                &mut p2[page.p2_index()],
287                p1_page,
288                parent_table_flags,
289                allocator,
290            )?
291        };
292
293        if !p1[page.p1_index()].is_unused() {
294            return Err(MapToError::PageAlreadyMapped(frame));
295        }
296        p1[page.p1_index()].set_frame(frame, flags);
297
298        Ok(MapperFlush::new(page))
299    }
300}
301
302impl Mapper<Size1GiB> for RecursivePageTable<'_> {
303    #[inline]
304    unsafe fn map_to_with_table_flags<A>(
305        &mut self,
306        page: Page<Size1GiB>,
307        frame: PhysFrame<Size1GiB>,
308        flags: PageTableFlags,
309        parent_table_flags: PageTableFlags,
310        allocator: &mut A,
311    ) -> Result<MapperFlush<Size1GiB>, MapToError<Size1GiB>>
312    where
313        A: FrameAllocator<Size4KiB> + ?Sized,
314    {
315        self.map_to_1gib(page, frame, flags, parent_table_flags, allocator)
316    }
317
318    fn unmap(
319        &mut self,
320        page: Page<Size1GiB>,
321    ) -> Result<(PhysFrame<Size1GiB>, MapperFlush<Size1GiB>), UnmapError> {
322        let p4 = &mut self.p4;
323        let p4_entry = &p4[page.p4_index()];
324
325        p4_entry.frame().map_err(|err| match err {
326            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
327            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
328        })?;
329
330        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
331        let p3_entry = &mut p3[page.p3_index()];
332        let flags = p3_entry.flags();
333
334        if !flags.contains(PageTableFlags::PRESENT) {
335            return Err(UnmapError::PageNotMapped);
336        }
337        if !flags.contains(PageTableFlags::HUGE_PAGE) {
338            return Err(UnmapError::ParentEntryHugePage);
339        }
340
341        let frame = PhysFrame::from_start_address(p3_entry.addr())
342            .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(p3_entry.addr()))?;
343
344        p3_entry.set_unused();
345        Ok((frame, MapperFlush::new(page)))
346    }
347
348    unsafe fn update_flags(
349        &mut self,
350        page: Page<Size1GiB>,
351        flags: PageTableFlags,
352    ) -> Result<MapperFlush<Size1GiB>, FlagUpdateError> {
353        use crate::structures::paging::PageTableFlags as Flags;
354        let p4 = &mut self.p4;
355
356        if p4[page.p4_index()].is_unused() {
357            return Err(FlagUpdateError::PageNotMapped);
358        }
359
360        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
361
362        if p3[page.p3_index()].is_unused() {
363            return Err(FlagUpdateError::PageNotMapped);
364        }
365        p3[page.p3_index()].set_flags(flags | Flags::HUGE_PAGE);
366
367        Ok(MapperFlush::new(page))
368    }
369
370    unsafe fn set_flags_p4_entry(
371        &mut self,
372        page: Page<Size1GiB>,
373        flags: PageTableFlags,
374    ) -> Result<MapperFlushAll, FlagUpdateError> {
375        let p4 = &mut self.p4;
376        let p4_entry = &mut p4[page.p4_index()];
377
378        if p4_entry.is_unused() {
379            return Err(FlagUpdateError::PageNotMapped);
380        }
381
382        p4_entry.set_flags(flags);
383
384        Ok(MapperFlushAll::new())
385    }
386
387    unsafe fn set_flags_p3_entry(
388        &mut self,
389        _page: Page<Size1GiB>,
390        _flags: PageTableFlags,
391    ) -> Result<MapperFlushAll, FlagUpdateError> {
392        Err(FlagUpdateError::ParentEntryHugePage)
393    }
394
395    unsafe fn set_flags_p2_entry(
396        &mut self,
397        _page: Page<Size1GiB>,
398        _flags: PageTableFlags,
399    ) -> Result<MapperFlushAll, FlagUpdateError> {
400        Err(FlagUpdateError::ParentEntryHugePage)
401    }
402
403    fn translate_page(&self, page: Page<Size1GiB>) -> Result<PhysFrame<Size1GiB>, TranslateError> {
404        let p4 = &self.p4;
405
406        if p4[page.p4_index()].is_unused() {
407            return Err(TranslateError::PageNotMapped);
408        }
409
410        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
411        let p3_entry = &p3[page.p3_index()];
412
413        if p3_entry.is_unused() {
414            return Err(TranslateError::PageNotMapped);
415        }
416
417        PhysFrame::from_start_address(p3_entry.addr())
418            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p3_entry.addr()))
419    }
420}
421
422impl Mapper<Size2MiB> for RecursivePageTable<'_> {
423    #[inline]
424    unsafe fn map_to_with_table_flags<A>(
425        &mut self,
426        page: Page<Size2MiB>,
427        frame: PhysFrame<Size2MiB>,
428        flags: PageTableFlags,
429        parent_table_flags: PageTableFlags,
430        allocator: &mut A,
431    ) -> Result<MapperFlush<Size2MiB>, MapToError<Size2MiB>>
432    where
433        A: FrameAllocator<Size4KiB> + ?Sized,
434    {
435        self.map_to_2mib(page, frame, flags, parent_table_flags, allocator)
436    }
437
438    fn unmap(
439        &mut self,
440        page: Page<Size2MiB>,
441    ) -> Result<(PhysFrame<Size2MiB>, MapperFlush<Size2MiB>), UnmapError> {
442        let p4 = &mut self.p4;
443        let p4_entry = &p4[page.p4_index()];
444        p4_entry.frame().map_err(|err| match err {
445            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
446            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
447        })?;
448
449        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
450        let p3_entry = &p3[page.p3_index()];
451        p3_entry.frame().map_err(|err| match err {
452            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
453            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
454        })?;
455
456        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
457        let p2_entry = &mut p2[page.p2_index()];
458        let flags = p2_entry.flags();
459
460        if !flags.contains(PageTableFlags::PRESENT) {
461            return Err(UnmapError::PageNotMapped);
462        }
463        if !flags.contains(PageTableFlags::HUGE_PAGE) {
464            return Err(UnmapError::ParentEntryHugePage);
465        }
466
467        let frame = PhysFrame::from_start_address(p2_entry.addr())
468            .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(p2_entry.addr()))?;
469
470        p2_entry.set_unused();
471        Ok((frame, MapperFlush::new(page)))
472    }
473
474    unsafe fn update_flags(
475        &mut self,
476        page: Page<Size2MiB>,
477        flags: PageTableFlags,
478    ) -> Result<MapperFlush<Size2MiB>, FlagUpdateError> {
479        use crate::structures::paging::PageTableFlags as Flags;
480        let p4 = &mut self.p4;
481
482        if p4[page.p4_index()].is_unused() {
483            return Err(FlagUpdateError::PageNotMapped);
484        }
485
486        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
487
488        if p3[page.p3_index()].is_unused() {
489            return Err(FlagUpdateError::PageNotMapped);
490        }
491
492        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
493
494        if p2[page.p2_index()].is_unused() {
495            return Err(FlagUpdateError::PageNotMapped);
496        }
497
498        p2[page.p2_index()].set_flags(flags | Flags::HUGE_PAGE);
499
500        Ok(MapperFlush::new(page))
501    }
502
503    unsafe fn set_flags_p4_entry(
504        &mut self,
505        page: Page<Size2MiB>,
506        flags: PageTableFlags,
507    ) -> Result<MapperFlushAll, FlagUpdateError> {
508        let p4 = &mut self.p4;
509        let p4_entry = &mut p4[page.p4_index()];
510
511        if p4_entry.is_unused() {
512            return Err(FlagUpdateError::PageNotMapped);
513        }
514
515        p4_entry.set_flags(flags);
516
517        Ok(MapperFlushAll::new())
518    }
519
520    unsafe fn set_flags_p3_entry(
521        &mut self,
522        page: Page<Size2MiB>,
523        flags: PageTableFlags,
524    ) -> Result<MapperFlushAll, FlagUpdateError> {
525        let p4 = &mut self.p4;
526
527        if p4[page.p4_index()].is_unused() {
528            return Err(FlagUpdateError::PageNotMapped);
529        }
530
531        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
532        let p3_entry = &mut p3[page.p3_index()];
533
534        if p3_entry.is_unused() {
535            return Err(FlagUpdateError::PageNotMapped);
536        }
537
538        p3_entry.set_flags(flags);
539
540        Ok(MapperFlushAll::new())
541    }
542
543    unsafe fn set_flags_p2_entry(
544        &mut self,
545        _page: Page<Size2MiB>,
546        _flags: PageTableFlags,
547    ) -> Result<MapperFlushAll, FlagUpdateError> {
548        Err(FlagUpdateError::ParentEntryHugePage)
549    }
550
551    fn translate_page(&self, page: Page<Size2MiB>) -> Result<PhysFrame<Size2MiB>, TranslateError> {
552        let p4 = &self.p4;
553
554        if p4[page.p4_index()].is_unused() {
555            return Err(TranslateError::PageNotMapped);
556        }
557
558        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
559        let p3_entry = &p3[page.p3_index()];
560
561        if p3_entry.is_unused() {
562            return Err(TranslateError::PageNotMapped);
563        }
564
565        let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
566        let p2_entry = &p2[page.p2_index()];
567
568        if p2_entry.is_unused() {
569            return Err(TranslateError::PageNotMapped);
570        }
571
572        PhysFrame::from_start_address(p2_entry.addr())
573            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p2_entry.addr()))
574    }
575}
576
577impl Mapper<Size4KiB> for RecursivePageTable<'_> {
578    #[inline]
579    unsafe fn map_to_with_table_flags<A>(
580        &mut self,
581        page: Page<Size4KiB>,
582        frame: PhysFrame<Size4KiB>,
583        flags: PageTableFlags,
584        parent_table_flags: PageTableFlags,
585        allocator: &mut A,
586    ) -> Result<MapperFlush<Size4KiB>, MapToError<Size4KiB>>
587    where
588        A: FrameAllocator<Size4KiB> + ?Sized,
589    {
590        self.map_to_4kib(page, frame, flags, parent_table_flags, allocator)
591    }
592
593    fn unmap(
594        &mut self,
595        page: Page<Size4KiB>,
596    ) -> Result<(PhysFrame<Size4KiB>, MapperFlush<Size4KiB>), UnmapError> {
597        let p4 = &mut self.p4;
598        let p4_entry = &p4[page.p4_index()];
599        p4_entry.frame().map_err(|err| match err {
600            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
601            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
602        })?;
603
604        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
605        let p3_entry = &p3[page.p3_index()];
606        p3_entry.frame().map_err(|err| match err {
607            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
608            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
609        })?;
610
611        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
612        let p2_entry = &p2[page.p2_index()];
613        p2_entry.frame().map_err(|err| match err {
614            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
615            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
616        })?;
617
618        let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) };
619        let p1_entry = &mut p1[page.p1_index()];
620
621        let frame = p1_entry.frame().map_err(|err| match err {
622            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
623            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
624        })?;
625
626        p1_entry.set_unused();
627        Ok((frame, MapperFlush::new(page)))
628    }
629
630    unsafe fn update_flags(
631        &mut self,
632        page: Page<Size4KiB>,
633        flags: PageTableFlags,
634    ) -> Result<MapperFlush<Size4KiB>, FlagUpdateError> {
635        let p4 = &mut self.p4;
636
637        if p4[page.p4_index()].is_unused() {
638            return Err(FlagUpdateError::PageNotMapped);
639        }
640
641        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
642
643        if p3[page.p3_index()].is_unused() {
644            return Err(FlagUpdateError::PageNotMapped);
645        }
646
647        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
648
649        if p2[page.p2_index()].is_unused() {
650            return Err(FlagUpdateError::PageNotMapped);
651        }
652
653        let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) };
654
655        if p1[page.p1_index()].is_unused() {
656            return Err(FlagUpdateError::PageNotMapped);
657        }
658
659        p1[page.p1_index()].set_flags(flags);
660
661        Ok(MapperFlush::new(page))
662    }
663
664    unsafe fn set_flags_p4_entry(
665        &mut self,
666        page: Page<Size4KiB>,
667        flags: PageTableFlags,
668    ) -> Result<MapperFlushAll, FlagUpdateError> {
669        let p4 = &mut self.p4;
670        let p4_entry = &mut p4[page.p4_index()];
671
672        if p4_entry.is_unused() {
673            return Err(FlagUpdateError::PageNotMapped);
674        }
675
676        p4_entry.set_flags(flags);
677
678        Ok(MapperFlushAll::new())
679    }
680
681    unsafe fn set_flags_p3_entry(
682        &mut self,
683        page: Page<Size4KiB>,
684        flags: PageTableFlags,
685    ) -> Result<MapperFlushAll, FlagUpdateError> {
686        let p4 = &mut self.p4;
687
688        if p4[page.p4_index()].is_unused() {
689            return Err(FlagUpdateError::PageNotMapped);
690        }
691
692        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
693        let p3_entry = &mut p3[page.p3_index()];
694
695        if p3_entry.is_unused() {
696            return Err(FlagUpdateError::PageNotMapped);
697        }
698
699        p3_entry.set_flags(flags);
700
701        Ok(MapperFlushAll::new())
702    }
703
704    unsafe fn set_flags_p2_entry(
705        &mut self,
706        page: Page<Size4KiB>,
707        flags: PageTableFlags,
708    ) -> Result<MapperFlushAll, FlagUpdateError> {
709        let p4 = &mut self.p4;
710
711        if p4[page.p4_index()].is_unused() {
712            return Err(FlagUpdateError::PageNotMapped);
713        }
714
715        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
716
717        if p3[page.p3_index()].is_unused() {
718            return Err(FlagUpdateError::PageNotMapped);
719        }
720
721        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
722        let p2_entry = &mut p2[page.p2_index()];
723
724        if p2_entry.is_unused() {
725            return Err(FlagUpdateError::PageNotMapped);
726        }
727
728        p2_entry.set_flags(flags);
729
730        Ok(MapperFlushAll::new())
731    }
732
733    fn translate_page(&self, page: Page<Size4KiB>) -> Result<PhysFrame<Size4KiB>, TranslateError> {
734        let p4 = &self.p4;
735
736        if p4[page.p4_index()].is_unused() {
737            return Err(TranslateError::PageNotMapped);
738        }
739
740        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
741        let p3_entry = &p3[page.p3_index()];
742
743        if p3_entry.is_unused() {
744            return Err(TranslateError::PageNotMapped);
745        }
746
747        let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
748        let p2_entry = &p2[page.p2_index()];
749
750        if p2_entry.is_unused() {
751            return Err(TranslateError::PageNotMapped);
752        }
753
754        let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) };
755        let p1_entry = &p1[page.p1_index()];
756
757        if p1_entry.is_unused() {
758            return Err(TranslateError::PageNotMapped);
759        }
760
761        PhysFrame::from_start_address(p1_entry.addr())
762            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p1_entry.addr()))
763    }
764}
765
766impl Translate for RecursivePageTable<'_> {
767    #[allow(clippy::inconsistent_digit_grouping)]
768    fn translate(&self, addr: VirtAddr) -> TranslateResult {
769        let page = Page::containing_address(addr);
770
771        let p4 = &self.p4;
772        let p4_entry = &p4[addr.p4_index()];
773        if p4_entry.is_unused() {
774            return TranslateResult::NotMapped;
775        }
776        if p4_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
777            panic!("level 4 entry has huge page bit set")
778        }
779
780        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
781        let p3_entry = &p3[addr.p3_index()];
782        if p3_entry.is_unused() {
783            return TranslateResult::NotMapped;
784        }
785        if p3_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
786            let entry = &p3[addr.p3_index()];
787            let frame = PhysFrame::containing_address(entry.addr());
788            #[allow(clippy::unusual_byte_groupings)]
789            let offset = addr.as_u64() & 0o_777_777_7777;
790            let flags = entry.flags();
791            return TranslateResult::Mapped {
792                frame: MappedFrame::Size1GiB(frame),
793                offset,
794                flags,
795            };
796        }
797
798        let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
799        let p2_entry = &p2[addr.p2_index()];
800        if p2_entry.is_unused() {
801            return TranslateResult::NotMapped;
802        }
803        if p2_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
804            let entry = &p2[addr.p2_index()];
805            let frame = PhysFrame::containing_address(entry.addr());
806            #[allow(clippy::unusual_byte_groupings)]
807            let offset = addr.as_u64() & 0o_777_7777;
808            let flags = entry.flags();
809            return TranslateResult::Mapped {
810                frame: MappedFrame::Size2MiB(frame),
811                offset,
812                flags,
813            };
814        }
815
816        let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) };
817        let p1_entry = &p1[addr.p1_index()];
818        if p1_entry.is_unused() {
819            return TranslateResult::NotMapped;
820        }
821        if p1_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
822            panic!("level 1 entry has huge page bit set")
823        }
824
825        let frame = match PhysFrame::from_start_address(p1_entry.addr()) {
826            Ok(frame) => frame,
827            Err(AddressNotAligned) => return TranslateResult::InvalidFrameAddress(p1_entry.addr()),
828        };
829        let offset = u64::from(addr.page_offset());
830        let flags = p1_entry.flags();
831        TranslateResult::Mapped {
832            frame: MappedFrame::Size4KiB(frame),
833            offset,
834            flags,
835        }
836    }
837}
838
839impl CleanUp for RecursivePageTable<'_> {
840    #[inline]
841    unsafe fn clean_up<D>(&mut self, frame_deallocator: &mut D)
842    where
843        D: FrameDeallocator<Size4KiB>,
844    {
845        unsafe {
846            self.clean_up_addr_range(
847                PageRangeInclusive {
848                    start: Page::from_start_address(VirtAddr::new(0)).unwrap(),
849                    end: Page::from_start_address(VirtAddr::new(0xffff_ffff_ffff_f000)).unwrap(),
850                },
851                frame_deallocator,
852            )
853        }
854    }
855
856    unsafe fn clean_up_addr_range<D>(
857        &mut self,
858        range: PageRangeInclusive,
859        frame_deallocator: &mut D,
860    ) where
861        D: FrameDeallocator<Size4KiB>,
862    {
863        fn clean_up(
864            recursive_index: PageTableIndex,
865            page_table: &mut PageTable,
866            level: PageTableLevel,
867            range: PageRangeInclusive,
868            frame_deallocator: &mut impl FrameDeallocator<Size4KiB>,
869        ) -> bool {
870            if range.is_empty() {
871                return false;
872            }
873
874            let table_addr = range
875                .start
876                .start_address()
877                .align_down(level.table_address_space_alignment());
878
879            let start = range.start.page_table_index(level);
880            let end = range.end.page_table_index(level);
881
882            if let Some(next_level) = level.next_lower_level() {
883                let offset_per_entry = level.entry_address_space_alignment();
884                for (i, entry) in page_table
885                    .iter_mut()
886                    .enumerate()
887                    .take(usize::from(end) + 1)
888                    .skip(usize::from(start))
889                    .filter(|(i, _)| {
890                        !(level == PageTableLevel::Four && *i == recursive_index.into())
891                    })
892                {
893                    if let Ok(frame) = entry.frame() {
894                        let start = VirtAddr::forward_checked_impl(
895                            table_addr,
896                            (offset_per_entry as usize) * i,
897                        )
898                        .unwrap();
899                        let end = start + (offset_per_entry - 1);
900                        let start = Page::<Size4KiB>::containing_address(start);
901                        let start = start.max(range.start);
902                        let end = Page::<Size4KiB>::containing_address(end);
903                        let end = end.min(range.end);
904                        let page_table =
905                            [p1_ptr, p2_ptr, p3_ptr][level as usize - 2](start, recursive_index);
906                        let page_table = unsafe { &mut *page_table };
907                        if clean_up(
908                            recursive_index,
909                            page_table,
910                            next_level,
911                            Page::range_inclusive(start, end),
912                            frame_deallocator,
913                        ) {
914                            entry.set_unused();
915                            unsafe {
916                                frame_deallocator.deallocate_frame(frame);
917                            }
918                        }
919                    }
920                }
921            }
922
923            page_table.iter().all(PageTableEntry::is_unused)
924        }
925
926        clean_up(
927            self.recursive_index,
928            self.level_4_table_mut(),
929            PageTableLevel::Four,
930            range,
931            frame_deallocator,
932        );
933    }
934}
935
936/// The given page table was not suitable to create a `RecursivePageTable`.
937#[derive(Debug)]
938pub enum InvalidPageTable {
939    /// The given page table was not at an recursive address.
940    ///
941    /// The page table address must be of the form `0o_xxx_xxx_xxx_xxx_0000` where `xxx`
942    /// is the recursive entry.
943    NotRecursive,
944    /// The given page table was not active on the CPU.
945    ///
946    /// The recursive page table design requires that the given level 4 table is active
947    /// on the CPU because otherwise it's not possible to access the other page tables
948    /// through recursive memory addresses.
949    NotActive,
950}
951
952impl fmt::Display for InvalidPageTable {
953    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
954        match self {
955            InvalidPageTable::NotRecursive => {
956                write!(f, "given page table address is not recursive")
957            }
958            InvalidPageTable::NotActive => write!(f, "given page table is not active on the CPU"),
959        }
960    }
961}
962
963#[inline]
964fn p3_ptr<S: PageSize>(page: Page<S>, recursive_index: PageTableIndex) -> *mut PageTable {
965    p3_page(page, recursive_index).start_address().as_mut_ptr()
966}
967
968#[inline]
969fn p3_page<S: PageSize>(page: Page<S>, recursive_index: PageTableIndex) -> Page {
970    Page::from_page_table_indices(
971        recursive_index,
972        recursive_index,
973        recursive_index,
974        page.p4_index(),
975    )
976}
977
978#[inline]
979fn p2_ptr<S: NotGiantPageSize>(page: Page<S>, recursive_index: PageTableIndex) -> *mut PageTable {
980    p2_page(page, recursive_index).start_address().as_mut_ptr()
981}
982
983#[inline]
984fn p2_page<S: NotGiantPageSize>(page: Page<S>, recursive_index: PageTableIndex) -> Page {
985    Page::from_page_table_indices(
986        recursive_index,
987        recursive_index,
988        page.p4_index(),
989        page.p3_index(),
990    )
991}
992
993#[inline]
994fn p1_ptr(page: Page<Size4KiB>, recursive_index: PageTableIndex) -> *mut PageTable {
995    p1_page(page, recursive_index).start_address().as_mut_ptr()
996}
997
998#[inline]
999fn p1_page(page: Page<Size4KiB>, recursive_index: PageTableIndex) -> Page {
1000    Page::from_page_table_indices(
1001        recursive_index,
1002        page.p4_index(),
1003        page.p3_index(),
1004        page.p2_index(),
1005    )
1006}
x86_64/structures/paging/mapper/recursive_page_table.rs

x86_64/structures/paging/mapper/
recursive_page_table.rs
