x86_64/structures/paging/mapper/
mapped_page_table.rs

1use crate::structures::paging::{
2    mapper::*,
3    page::AddressNotAligned,
4    page_table::{FrameError, PageTable, PageTableEntry, PageTableLevel},
5};
6
7/// A Mapper implementation that relies on a PhysAddr to VirtAddr conversion function.
8///
9/// This type requires that the all physical page table frames are mapped to some virtual
10/// address. Normally, this is done by mapping the complete physical address space into
11/// the virtual address space at some offset. Other mappings between physical and virtual
12/// memory are possible too, as long as they can be calculated as an `PhysAddr` to
13/// `VirtAddr` closure.
14#[derive(Debug)]
15pub struct MappedPageTable<'a, P: PageTableFrameMapping> {
16    page_table_walker: PageTableWalker<P>,
17    level_4_table: &'a mut PageTable,
18}
19
20impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> {
21    /// Creates a new `MappedPageTable` that uses the passed `PageTableFrameMapping` for converting virtual
22    /// to physical addresses.
23    ///
24    /// ## Safety
25    ///
26    /// This function is unsafe because the caller must guarantee that the passed `page_table_frame_mapping`
27    /// `PageTableFrameMapping` is correct. Also, the passed `level_4_table` must point to the level 4 page table
28    /// of a valid page table hierarchy. Otherwise this function might break memory safety, e.g.
29    /// by writing to an illegal memory location.
30    #[inline]
31    pub unsafe fn new(level_4_table: &'a mut PageTable, page_table_frame_mapping: P) -> Self {
32        Self {
33            level_4_table,
34            page_table_walker: unsafe { PageTableWalker::new(page_table_frame_mapping) },
35        }
36    }
37
38    /// Returns an immutable reference to the wrapped level 4 `PageTable` instance.
39    pub fn level_4_table(&self) -> &PageTable {
40        self.level_4_table
41    }
42
43    /// Returns a mutable reference to the wrapped level 4 `PageTable` instance.
44    pub fn level_4_table_mut(&mut self) -> &mut PageTable {
45        self.level_4_table
46    }
47
48    /// Returns the `PageTableFrameMapping` used for converting virtual to physical addresses.
49    pub fn page_table_frame_mapping(&self) -> &P {
50        &self.page_table_walker.page_table_frame_mapping
51    }
52
53    /// Helper function for implementing Mapper. Safe to limit the scope of unsafe, see
54    /// https://github.com/rust-lang/rfcs/pull/2585.
55    fn map_to_1gib<A>(
56        &mut self,
57        page: Page<Size1GiB>,
58        frame: PhysFrame<Size1GiB>,
59        flags: PageTableFlags,
60        parent_table_flags: PageTableFlags,
61        allocator: &mut A,
62    ) -> Result<MapperFlush<Size1GiB>, MapToError<Size1GiB>>
63    where
64        A: FrameAllocator<Size4KiB> + ?Sized,
65    {
66        let p4 = &mut self.level_4_table;
67        let p3 = self.page_table_walker.create_next_table(
68            &mut p4[page.p4_index()],
69            parent_table_flags,
70            allocator,
71        )?;
72
73        if !p3[page.p3_index()].is_unused() {
74            return Err(MapToError::PageAlreadyMapped(frame));
75        }
76        p3[page.p3_index()].set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE);
77
78        Ok(MapperFlush::new(page))
79    }
80
81    /// Helper function for implementing Mapper. Safe to limit the scope of unsafe, see
82    /// https://github.com/rust-lang/rfcs/pull/2585.
83    fn map_to_2mib<A>(
84        &mut self,
85        page: Page<Size2MiB>,
86        frame: PhysFrame<Size2MiB>,
87        flags: PageTableFlags,
88        parent_table_flags: PageTableFlags,
89        allocator: &mut A,
90    ) -> Result<MapperFlush<Size2MiB>, MapToError<Size2MiB>>
91    where
92        A: FrameAllocator<Size4KiB> + ?Sized,
93    {
94        let p4 = &mut self.level_4_table;
95        let p3 = self.page_table_walker.create_next_table(
96            &mut p4[page.p4_index()],
97            parent_table_flags,
98            allocator,
99        )?;
100        let p2 = self.page_table_walker.create_next_table(
101            &mut p3[page.p3_index()],
102            parent_table_flags,
103            allocator,
104        )?;
105
106        if !p2[page.p2_index()].is_unused() {
107            return Err(MapToError::PageAlreadyMapped(frame));
108        }
109        p2[page.p2_index()].set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE);
110
111        Ok(MapperFlush::new(page))
112    }
113
114    /// Helper function for implementing Mapper. Safe to limit the scope of unsafe, see
115    /// https://github.com/rust-lang/rfcs/pull/2585.
116    fn map_to_4kib<A>(
117        &mut self,
118        page: Page<Size4KiB>,
119        frame: PhysFrame<Size4KiB>,
120        flags: PageTableFlags,
121        parent_table_flags: PageTableFlags,
122        allocator: &mut A,
123    ) -> Result<MapperFlush<Size4KiB>, MapToError<Size4KiB>>
124    where
125        A: FrameAllocator<Size4KiB> + ?Sized,
126    {
127        let p4 = &mut self.level_4_table;
128        let p3 = self.page_table_walker.create_next_table(
129            &mut p4[page.p4_index()],
130            parent_table_flags,
131            allocator,
132        )?;
133        let p2 = self.page_table_walker.create_next_table(
134            &mut p3[page.p3_index()],
135            parent_table_flags,
136            allocator,
137        )?;
138        let p1 = self.page_table_walker.create_next_table(
139            &mut p2[page.p2_index()],
140            parent_table_flags,
141            allocator,
142        )?;
143
144        if !p1[page.p1_index()].is_unused() {
145            return Err(MapToError::PageAlreadyMapped(frame));
146        }
147        p1[page.p1_index()].set_frame(frame, flags);
148
149        Ok(MapperFlush::new(page))
150    }
151}
152
153impl<P: PageTableFrameMapping> Mapper<Size1GiB> for MappedPageTable<'_, P> {
154    #[inline]
155    unsafe fn map_to_with_table_flags<A>(
156        &mut self,
157        page: Page<Size1GiB>,
158        frame: PhysFrame<Size1GiB>,
159        flags: PageTableFlags,
160        parent_table_flags: PageTableFlags,
161        allocator: &mut A,
162    ) -> Result<MapperFlush<Size1GiB>, MapToError<Size1GiB>>
163    where
164        A: FrameAllocator<Size4KiB> + ?Sized,
165    {
166        self.map_to_1gib(page, frame, flags, parent_table_flags, allocator)
167    }
168
169    fn unmap(
170        &mut self,
171        page: Page<Size1GiB>,
172    ) -> Result<(PhysFrame<Size1GiB>, MapperFlush<Size1GiB>), UnmapError> {
173        let p4 = &mut self.level_4_table;
174        let p3 = self
175            .page_table_walker
176            .next_table_mut(&mut p4[page.p4_index()])?;
177
178        let p3_entry = &mut p3[page.p3_index()];
179        let flags = p3_entry.flags();
180
181        if !flags.contains(PageTableFlags::PRESENT) {
182            return Err(UnmapError::PageNotMapped);
183        }
184        if !flags.contains(PageTableFlags::HUGE_PAGE) {
185            return Err(UnmapError::ParentEntryHugePage);
186        }
187
188        let frame = PhysFrame::from_start_address(p3_entry.addr())
189            .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(p3_entry.addr()))?;
190
191        p3_entry.set_unused();
192        Ok((frame, MapperFlush::new(page)))
193    }
194
195    unsafe fn update_flags(
196        &mut self,
197        page: Page<Size1GiB>,
198        flags: PageTableFlags,
199    ) -> Result<MapperFlush<Size1GiB>, FlagUpdateError> {
200        let p4 = &mut self.level_4_table;
201        let p3 = self
202            .page_table_walker
203            .next_table_mut(&mut p4[page.p4_index()])?;
204
205        if p3[page.p3_index()].is_unused() {
206            return Err(FlagUpdateError::PageNotMapped);
207        }
208        p3[page.p3_index()].set_flags(flags | PageTableFlags::HUGE_PAGE);
209
210        Ok(MapperFlush::new(page))
211    }
212
213    unsafe fn set_flags_p4_entry(
214        &mut self,
215        page: Page<Size1GiB>,
216        flags: PageTableFlags,
217    ) -> Result<MapperFlushAll, FlagUpdateError> {
218        let p4 = &mut self.level_4_table;
219        let p4_entry = &mut p4[page.p4_index()];
220
221        if p4_entry.is_unused() {
222            return Err(FlagUpdateError::PageNotMapped);
223        }
224
225        p4_entry.set_flags(flags);
226
227        Ok(MapperFlushAll::new())
228    }
229
230    unsafe fn set_flags_p3_entry(
231        &mut self,
232        _page: Page<Size1GiB>,
233        _flags: PageTableFlags,
234    ) -> Result<MapperFlushAll, FlagUpdateError> {
235        Err(FlagUpdateError::ParentEntryHugePage)
236    }
237
238    unsafe fn set_flags_p2_entry(
239        &mut self,
240        _page: Page<Size1GiB>,
241        _flags: PageTableFlags,
242    ) -> Result<MapperFlushAll, FlagUpdateError> {
243        Err(FlagUpdateError::ParentEntryHugePage)
244    }
245
246    fn translate_page(&self, page: Page<Size1GiB>) -> Result<PhysFrame<Size1GiB>, TranslateError> {
247        let p4 = &self.level_4_table;
248        let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?;
249
250        let p3_entry = &p3[page.p3_index()];
251
252        if p3_entry.is_unused() {
253            return Err(TranslateError::PageNotMapped);
254        }
255
256        PhysFrame::from_start_address(p3_entry.addr())
257            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p3_entry.addr()))
258    }
259}
260
261impl<P: PageTableFrameMapping> Mapper<Size2MiB> for MappedPageTable<'_, P> {
262    #[inline]
263    unsafe fn map_to_with_table_flags<A>(
264        &mut self,
265        page: Page<Size2MiB>,
266        frame: PhysFrame<Size2MiB>,
267        flags: PageTableFlags,
268        parent_table_flags: PageTableFlags,
269        allocator: &mut A,
270    ) -> Result<MapperFlush<Size2MiB>, MapToError<Size2MiB>>
271    where
272        A: FrameAllocator<Size4KiB> + ?Sized,
273    {
274        self.map_to_2mib(page, frame, flags, parent_table_flags, allocator)
275    }
276
277    fn unmap(
278        &mut self,
279        page: Page<Size2MiB>,
280    ) -> Result<(PhysFrame<Size2MiB>, MapperFlush<Size2MiB>), UnmapError> {
281        let p4 = &mut self.level_4_table;
282        let p3 = self
283            .page_table_walker
284            .next_table_mut(&mut p4[page.p4_index()])?;
285        let p2 = self
286            .page_table_walker
287            .next_table_mut(&mut p3[page.p3_index()])?;
288
289        let p2_entry = &mut p2[page.p2_index()];
290        let flags = p2_entry.flags();
291
292        if !flags.contains(PageTableFlags::PRESENT) {
293            return Err(UnmapError::PageNotMapped);
294        }
295        if !flags.contains(PageTableFlags::HUGE_PAGE) {
296            return Err(UnmapError::ParentEntryHugePage);
297        }
298
299        let frame = PhysFrame::from_start_address(p2_entry.addr())
300            .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(p2_entry.addr()))?;
301
302        p2_entry.set_unused();
303        Ok((frame, MapperFlush::new(page)))
304    }
305
306    unsafe fn update_flags(
307        &mut self,
308        page: Page<Size2MiB>,
309        flags: PageTableFlags,
310    ) -> Result<MapperFlush<Size2MiB>, FlagUpdateError> {
311        let p4 = &mut self.level_4_table;
312        let p3 = self
313            .page_table_walker
314            .next_table_mut(&mut p4[page.p4_index()])?;
315        let p2 = self
316            .page_table_walker
317            .next_table_mut(&mut p3[page.p3_index()])?;
318
319        if p2[page.p2_index()].is_unused() {
320            return Err(FlagUpdateError::PageNotMapped);
321        }
322
323        p2[page.p2_index()].set_flags(flags | PageTableFlags::HUGE_PAGE);
324
325        Ok(MapperFlush::new(page))
326    }
327
328    unsafe fn set_flags_p4_entry(
329        &mut self,
330        page: Page<Size2MiB>,
331        flags: PageTableFlags,
332    ) -> Result<MapperFlushAll, FlagUpdateError> {
333        let p4 = &mut self.level_4_table;
334        let p4_entry = &mut p4[page.p4_index()];
335
336        if p4_entry.is_unused() {
337            return Err(FlagUpdateError::PageNotMapped);
338        }
339
340        p4_entry.set_flags(flags);
341
342        Ok(MapperFlushAll::new())
343    }
344
345    unsafe fn set_flags_p3_entry(
346        &mut self,
347        page: Page<Size2MiB>,
348        flags: PageTableFlags,
349    ) -> Result<MapperFlushAll, FlagUpdateError> {
350        let p4 = &mut self.level_4_table;
351        let p3 = self
352            .page_table_walker
353            .next_table_mut(&mut p4[page.p4_index()])?;
354        let p3_entry = &mut p3[page.p3_index()];
355
356        if p3_entry.is_unused() {
357            return Err(FlagUpdateError::PageNotMapped);
358        }
359
360        p3_entry.set_flags(flags);
361
362        Ok(MapperFlushAll::new())
363    }
364
365    unsafe fn set_flags_p2_entry(
366        &mut self,
367        _page: Page<Size2MiB>,
368        _flags: PageTableFlags,
369    ) -> Result<MapperFlushAll, FlagUpdateError> {
370        Err(FlagUpdateError::ParentEntryHugePage)
371    }
372
373    fn translate_page(&self, page: Page<Size2MiB>) -> Result<PhysFrame<Size2MiB>, TranslateError> {
374        let p4 = &self.level_4_table;
375        let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?;
376        let p2 = self.page_table_walker.next_table(&p3[page.p3_index()])?;
377
378        let p2_entry = &p2[page.p2_index()];
379
380        if p2_entry.is_unused() {
381            return Err(TranslateError::PageNotMapped);
382        }
383
384        PhysFrame::from_start_address(p2_entry.addr())
385            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p2_entry.addr()))
386    }
387}
388
389impl<P: PageTableFrameMapping> Mapper<Size4KiB> for MappedPageTable<'_, P> {
390    #[inline]
391    unsafe fn map_to_with_table_flags<A>(
392        &mut self,
393        page: Page<Size4KiB>,
394        frame: PhysFrame<Size4KiB>,
395        flags: PageTableFlags,
396        parent_table_flags: PageTableFlags,
397        allocator: &mut A,
398    ) -> Result<MapperFlush<Size4KiB>, MapToError<Size4KiB>>
399    where
400        A: FrameAllocator<Size4KiB> + ?Sized,
401    {
402        self.map_to_4kib(page, frame, flags, parent_table_flags, allocator)
403    }
404
405    fn unmap(
406        &mut self,
407        page: Page<Size4KiB>,
408    ) -> Result<(PhysFrame<Size4KiB>, MapperFlush<Size4KiB>), UnmapError> {
409        let p4 = &mut self.level_4_table;
410        let p3 = self
411            .page_table_walker
412            .next_table_mut(&mut p4[page.p4_index()])?;
413        let p2 = self
414            .page_table_walker
415            .next_table_mut(&mut p3[page.p3_index()])?;
416        let p1 = self
417            .page_table_walker
418            .next_table_mut(&mut p2[page.p2_index()])?;
419
420        let p1_entry = &mut p1[page.p1_index()];
421
422        let frame = p1_entry.frame().map_err(|err| match err {
423            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
424            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
425        })?;
426
427        p1_entry.set_unused();
428        Ok((frame, MapperFlush::new(page)))
429    }
430
431    unsafe fn update_flags(
432        &mut self,
433        page: Page<Size4KiB>,
434        flags: PageTableFlags,
435    ) -> Result<MapperFlush<Size4KiB>, FlagUpdateError> {
436        let p4 = &mut self.level_4_table;
437        let p3 = self
438            .page_table_walker
439            .next_table_mut(&mut p4[page.p4_index()])?;
440        let p2 = self
441            .page_table_walker
442            .next_table_mut(&mut p3[page.p3_index()])?;
443        let p1 = self
444            .page_table_walker
445            .next_table_mut(&mut p2[page.p2_index()])?;
446
447        if p1[page.p1_index()].is_unused() {
448            return Err(FlagUpdateError::PageNotMapped);
449        }
450
451        p1[page.p1_index()].set_flags(flags);
452
453        Ok(MapperFlush::new(page))
454    }
455
456    unsafe fn set_flags_p4_entry(
457        &mut self,
458        page: Page<Size4KiB>,
459        flags: PageTableFlags,
460    ) -> Result<MapperFlushAll, FlagUpdateError> {
461        let p4 = &mut self.level_4_table;
462        let p4_entry = &mut p4[page.p4_index()];
463
464        if p4_entry.is_unused() {
465            return Err(FlagUpdateError::PageNotMapped);
466        }
467
468        p4_entry.set_flags(flags);
469
470        Ok(MapperFlushAll::new())
471    }
472
473    unsafe fn set_flags_p3_entry(
474        &mut self,
475        page: Page<Size4KiB>,
476        flags: PageTableFlags,
477    ) -> Result<MapperFlushAll, FlagUpdateError> {
478        let p4 = &mut self.level_4_table;
479        let p3 = self
480            .page_table_walker
481            .next_table_mut(&mut p4[page.p4_index()])?;
482        let p3_entry = &mut p3[page.p3_index()];
483
484        if p3_entry.is_unused() {
485            return Err(FlagUpdateError::PageNotMapped);
486        }
487
488        p3_entry.set_flags(flags);
489
490        Ok(MapperFlushAll::new())
491    }
492
493    unsafe fn set_flags_p2_entry(
494        &mut self,
495        page: Page<Size4KiB>,
496        flags: PageTableFlags,
497    ) -> Result<MapperFlushAll, FlagUpdateError> {
498        let p4 = &mut self.level_4_table;
499        let p3 = self
500            .page_table_walker
501            .next_table_mut(&mut p4[page.p4_index()])?;
502        let p2 = self
503            .page_table_walker
504            .next_table_mut(&mut p3[page.p3_index()])?;
505        let p2_entry = &mut p2[page.p2_index()];
506
507        if p2_entry.is_unused() {
508            return Err(FlagUpdateError::PageNotMapped);
509        }
510
511        p2_entry.set_flags(flags);
512
513        Ok(MapperFlushAll::new())
514    }
515
516    fn translate_page(&self, page: Page<Size4KiB>) -> Result<PhysFrame<Size4KiB>, TranslateError> {
517        let p4 = &self.level_4_table;
518        let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?;
519        let p2 = self.page_table_walker.next_table(&p3[page.p3_index()])?;
520        let p1 = self.page_table_walker.next_table(&p2[page.p2_index()])?;
521
522        let p1_entry = &p1[page.p1_index()];
523
524        if p1_entry.is_unused() {
525            return Err(TranslateError::PageNotMapped);
526        }
527
528        PhysFrame::from_start_address(p1_entry.addr())
529            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p1_entry.addr()))
530    }
531}
532
533impl<P: PageTableFrameMapping> Translate for MappedPageTable<'_, P> {
534    #[allow(clippy::inconsistent_digit_grouping)]
535    fn translate(&self, addr: VirtAddr) -> TranslateResult {
536        let p4 = &self.level_4_table;
537        let p3 = match self.page_table_walker.next_table(&p4[addr.p4_index()]) {
538            Ok(page_table) => page_table,
539            Err(PageTableWalkError::NotMapped) => return TranslateResult::NotMapped,
540            Err(PageTableWalkError::MappedToHugePage) => {
541                panic!("level 4 entry has huge page bit set")
542            }
543        };
544        let p2 = match self.page_table_walker.next_table(&p3[addr.p3_index()]) {
545            Ok(page_table) => page_table,
546            Err(PageTableWalkError::NotMapped) => return TranslateResult::NotMapped,
547            Err(PageTableWalkError::MappedToHugePage) => {
548                let entry = &p3[addr.p3_index()];
549                let frame = PhysFrame::containing_address(entry.addr());
550                #[allow(clippy::unusual_byte_groupings)]
551                let offset = addr.as_u64() & 0o_777_777_7777;
552                let flags = entry.flags();
553                return TranslateResult::Mapped {
554                    frame: MappedFrame::Size1GiB(frame),
555                    offset,
556                    flags,
557                };
558            }
559        };
560        let p1 = match self.page_table_walker.next_table(&p2[addr.p2_index()]) {
561            Ok(page_table) => page_table,
562            Err(PageTableWalkError::NotMapped) => return TranslateResult::NotMapped,
563            Err(PageTableWalkError::MappedToHugePage) => {
564                let entry = &p2[addr.p2_index()];
565                let frame = PhysFrame::containing_address(entry.addr());
566                #[allow(clippy::unusual_byte_groupings)]
567                let offset = addr.as_u64() & 0o_777_7777;
568                let flags = entry.flags();
569                return TranslateResult::Mapped {
570                    frame: MappedFrame::Size2MiB(frame),
571                    offset,
572                    flags,
573                };
574            }
575        };
576
577        let p1_entry = &p1[addr.p1_index()];
578
579        if p1_entry.is_unused() {
580            return TranslateResult::NotMapped;
581        }
582
583        let frame = match PhysFrame::from_start_address(p1_entry.addr()) {
584            Ok(frame) => frame,
585            Err(AddressNotAligned) => return TranslateResult::InvalidFrameAddress(p1_entry.addr()),
586        };
587        let offset = u64::from(addr.page_offset());
588        let flags = p1_entry.flags();
589        TranslateResult::Mapped {
590            frame: MappedFrame::Size4KiB(frame),
591            offset,
592            flags,
593        }
594    }
595}
596
597impl<P: PageTableFrameMapping> CleanUp for MappedPageTable<'_, P> {
598    #[inline]
599    unsafe fn clean_up<D>(&mut self, frame_deallocator: &mut D)
600    where
601        D: FrameDeallocator<Size4KiB>,
602    {
603        unsafe {
604            self.clean_up_addr_range(
605                PageRangeInclusive {
606                    start: Page::from_start_address(VirtAddr::new(0)).unwrap(),
607                    end: Page::from_start_address(VirtAddr::new(0xffff_ffff_ffff_f000)).unwrap(),
608                },
609                frame_deallocator,
610            )
611        }
612    }
613
614    unsafe fn clean_up_addr_range<D>(
615        &mut self,
616        range: PageRangeInclusive,
617        frame_deallocator: &mut D,
618    ) where
619        D: FrameDeallocator<Size4KiB>,
620    {
621        unsafe fn clean_up<P: PageTableFrameMapping>(
622            page_table: &mut PageTable,
623            page_table_walker: &PageTableWalker<P>,
624            level: PageTableLevel,
625            range: PageRangeInclusive,
626            frame_deallocator: &mut impl FrameDeallocator<Size4KiB>,
627        ) -> bool {
628            if range.is_empty() {
629                return false;
630            }
631
632            let table_addr = range
633                .start
634                .start_address()
635                .align_down(level.table_address_space_alignment());
636
637            let start = range.start.page_table_index(level);
638            let end = range.end.page_table_index(level);
639
640            if let Some(next_level) = level.next_lower_level() {
641                let offset_per_entry = level.entry_address_space_alignment();
642                for (i, entry) in page_table
643                    .iter_mut()
644                    .enumerate()
645                    .take(usize::from(end) + 1)
646                    .skip(usize::from(start))
647                {
648                    if let Ok(page_table) = page_table_walker.next_table_mut(entry) {
649                        let start = VirtAddr::forward_checked_impl(
650                            table_addr,
651                            (offset_per_entry as usize) * i,
652                        )
653                        .unwrap();
654                        let end = start + (offset_per_entry - 1);
655                        let start = Page::<Size4KiB>::containing_address(start);
656                        let start = start.max(range.start);
657                        let end = Page::<Size4KiB>::containing_address(end);
658                        let end = end.min(range.end);
659                        unsafe {
660                            if clean_up(
661                                page_table,
662                                page_table_walker,
663                                next_level,
664                                Page::range_inclusive(start, end),
665                                frame_deallocator,
666                            ) {
667                                let frame = entry.frame().unwrap();
668                                entry.set_unused();
669                                frame_deallocator.deallocate_frame(frame);
670                            }
671                        }
672                    }
673                }
674            }
675
676            page_table.iter().all(PageTableEntry::is_unused)
677        }
678
679        unsafe {
680            clean_up(
681                self.level_4_table,
682                &self.page_table_walker,
683                PageTableLevel::Four,
684                range,
685                frame_deallocator,
686            );
687        }
688    }
689}
690
691#[derive(Debug)]
692struct PageTableWalker<P: PageTableFrameMapping> {
693    page_table_frame_mapping: P,
694}
695
696impl<P: PageTableFrameMapping> PageTableWalker<P> {
697    #[inline]
698    pub unsafe fn new(page_table_frame_mapping: P) -> Self {
699        Self {
700            page_table_frame_mapping,
701        }
702    }
703
704    /// Internal helper function to get a reference to the page table of the next level.
705    ///
706    /// Returns `PageTableWalkError::NotMapped` if the entry is unused. Returns
707    /// `PageTableWalkError::MappedToHugePage` if the `HUGE_PAGE` flag is set
708    /// in the passed entry.
709    #[inline]
710    fn next_table<'b>(
711        &self,
712        entry: &'b PageTableEntry,
713    ) -> Result<&'b PageTable, PageTableWalkError> {
714        let page_table_ptr = self
715            .page_table_frame_mapping
716            .frame_to_pointer(entry.frame()?);
717        let page_table: &PageTable = unsafe { &*page_table_ptr };
718
719        Ok(page_table)
720    }
721
722    /// Internal helper function to get a mutable reference to the page table of the next level.
723    ///
724    /// Returns `PageTableWalkError::NotMapped` if the entry is unused. Returns
725    /// `PageTableWalkError::MappedToHugePage` if the `HUGE_PAGE` flag is set
726    /// in the passed entry.
727    #[inline]
728    fn next_table_mut<'b>(
729        &self,
730        entry: &'b mut PageTableEntry,
731    ) -> Result<&'b mut PageTable, PageTableWalkError> {
732        let page_table_ptr = self
733            .page_table_frame_mapping
734            .frame_to_pointer(entry.frame()?);
735        let page_table: &mut PageTable = unsafe { &mut *page_table_ptr };
736
737        Ok(page_table)
738    }
739
740    /// Internal helper function to create the page table of the next level if needed.
741    ///
742    /// If the passed entry is unused, a new frame is allocated from the given allocator, zeroed,
743    /// and the entry is updated to that address. If the passed entry is already mapped, the next
744    /// table is returned directly.
745    ///
746    /// Returns `MapToError::FrameAllocationFailed` if the entry is unused and the allocator
747    /// returned `None`. Returns `MapToError::ParentEntryHugePage` if the `HUGE_PAGE` flag is set
748    /// in the passed entry.
749    fn create_next_table<'b, A>(
750        &self,
751        entry: &'b mut PageTableEntry,
752        insert_flags: PageTableFlags,
753        allocator: &mut A,
754    ) -> Result<&'b mut PageTable, PageTableCreateError>
755    where
756        A: FrameAllocator<Size4KiB> + ?Sized,
757    {
758        let created;
759
760        if entry.is_unused() {
761            if let Some(frame) = allocator.allocate_frame() {
762                entry.set_frame(frame, insert_flags);
763                created = true;
764            } else {
765                return Err(PageTableCreateError::FrameAllocationFailed);
766            }
767        } else {
768            if !insert_flags.is_empty() && !entry.flags().contains(insert_flags) {
769                entry.set_flags(entry.flags() | insert_flags);
770            }
771            created = false;
772        }
773
774        let page_table = match self.next_table_mut(entry) {
775            Err(PageTableWalkError::MappedToHugePage) => {
776                return Err(PageTableCreateError::MappedToHugePage);
777            }
778            Err(PageTableWalkError::NotMapped) => panic!("entry should be mapped at this point"),
779            Ok(page_table) => page_table,
780        };
781
782        if created {
783            page_table.zero();
784        }
785        Ok(page_table)
786    }
787}
788
789#[derive(Debug)]
790enum PageTableWalkError {
791    NotMapped,
792    MappedToHugePage,
793}
794
795#[derive(Debug)]
796enum PageTableCreateError {
797    MappedToHugePage,
798    FrameAllocationFailed,
799}
800
801impl From<PageTableCreateError> for MapToError<Size4KiB> {
802    #[inline]
803    fn from(err: PageTableCreateError) -> Self {
804        match err {
805            PageTableCreateError::MappedToHugePage => MapToError::ParentEntryHugePage,
806            PageTableCreateError::FrameAllocationFailed => MapToError::FrameAllocationFailed,
807        }
808    }
809}
810
811impl From<PageTableCreateError> for MapToError<Size2MiB> {
812    #[inline]
813    fn from(err: PageTableCreateError) -> Self {
814        match err {
815            PageTableCreateError::MappedToHugePage => MapToError::ParentEntryHugePage,
816            PageTableCreateError::FrameAllocationFailed => MapToError::FrameAllocationFailed,
817        }
818    }
819}
820
821impl From<PageTableCreateError> for MapToError<Size1GiB> {
822    #[inline]
823    fn from(err: PageTableCreateError) -> Self {
824        match err {
825            PageTableCreateError::MappedToHugePage => MapToError::ParentEntryHugePage,
826            PageTableCreateError::FrameAllocationFailed => MapToError::FrameAllocationFailed,
827        }
828    }
829}
830
831impl From<FrameError> for PageTableWalkError {
832    #[inline]
833    fn from(err: FrameError) -> Self {
834        match err {
835            FrameError::HugeFrame => PageTableWalkError::MappedToHugePage,
836            FrameError::FrameNotPresent => PageTableWalkError::NotMapped,
837        }
838    }
839}
840
841impl From<PageTableWalkError> for UnmapError {
842    #[inline]
843    fn from(err: PageTableWalkError) -> Self {
844        match err {
845            PageTableWalkError::MappedToHugePage => UnmapError::ParentEntryHugePage,
846            PageTableWalkError::NotMapped => UnmapError::PageNotMapped,
847        }
848    }
849}
850
851impl From<PageTableWalkError> for FlagUpdateError {
852    #[inline]
853    fn from(err: PageTableWalkError) -> Self {
854        match err {
855            PageTableWalkError::MappedToHugePage => FlagUpdateError::ParentEntryHugePage,
856            PageTableWalkError::NotMapped => FlagUpdateError::PageNotMapped,
857        }
858    }
859}
860
861impl From<PageTableWalkError> for TranslateError {
862    #[inline]
863    fn from(err: PageTableWalkError) -> Self {
864        match err {
865            PageTableWalkError::MappedToHugePage => TranslateError::ParentEntryHugePage,
866            PageTableWalkError::NotMapped => TranslateError::PageNotMapped,
867        }
868    }
869}
870
871/// Provides a virtual address mapping for physical page table frames.
872///
873/// This only works if the physical address space is somehow mapped to the virtual
874/// address space, e.g. at an offset.
875///
876/// ## Safety
877///
878/// This trait is unsafe to implement because the implementer must ensure that
879/// `frame_to_pointer` returns a valid page table pointer for any given physical frame.
880pub unsafe trait PageTableFrameMapping {
881    /// Translate the given physical frame to a virtual page table pointer.
882    fn frame_to_pointer(&self, frame: PhysFrame) -> *mut PageTable;
883}
x86_64/structures/paging/mapper/mapped_page_table.rs

x86_64/structures/paging/mapper/
mapped_page_table.rs
