use super::{PhysicalMemRegion, FrameAllocatorTrait, FrameAllocatorTraitPrivate};
use crate::paging::PAGE_SIZE;
use multiboot2::BootInformation;
use crate::sync::SpinLock;
use alloc::vec::Vec;
use crate::utils::{check_size_aligned, check_nonzero_length};
use bit_field::BitArray;
use crate::utils::BitArrayExt;
use crate::mem::PhysicalAddress;
use crate::mem::{round_to_page, round_to_page_upper};
use crate::paging::kernel_memory::get_kernel_memory;
use crate::error::KernelError;
use failure::Backtrace;
const FRAME_OFFSET_MASK: usize = 0xFFF;
const FRAME_BASE_MASK: usize = !FRAME_OFFSET_MASK;
const FRAME_BASE_LOG: usize = 12;
#[cfg(not(any(test, doc)))]
const FRAMES_BITMAP_SIZE: usize = usize::max_value() / PAGE_SIZE / 8 + 1;
#[cfg(any(test, doc))]
const FRAMES_BITMAP_SIZE: usize = 32 / 8;
#[inline]
fn addr_to_frame(addr: usize) -> usize {
addr >> FRAME_BASE_LOG
}
#[inline]
fn frame_to_addr(frame: usize) -> usize {
frame << FRAME_BASE_LOG
}
pub struct FrameAllocatori386 {
memory_bitmap: [u8; FRAMES_BITMAP_SIZE],
initialized: bool
}
const FRAME_FREE: bool = true;
const FRAME_OCCUPIED: bool = false;
#[cfg_attr(test, thread_local)]
static FRAME_ALLOCATOR : SpinLock<FrameAllocatori386> = SpinLock::new(FrameAllocatori386::new());
impl FrameAllocatori386 {
pub const fn new() -> Self {
FrameAllocatori386 {
memory_bitmap: [0x00; FRAMES_BITMAP_SIZE],
initialized: false
}
}
}
#[derive(Debug)]
pub struct FrameAllocator;
impl FrameAllocatorTraitPrivate for FrameAllocator {
fn free_region(region: &PhysicalMemRegion) {
if region.frames > 0 {
debug!("Freeing {:?}", region);
assert!(Self::check_is_allocated(region.address(), region.size()), "PhysMemRegion beeing freed was not allocated");
let mut allocator = FRAME_ALLOCATOR.lock();
assert!(allocator.initialized, "The frame allocator was not initialized");
allocator.memory_bitmap.set_bits_area(
addr_to_frame(region.address().addr())
..
addr_to_frame(region.address().addr() + region.size()),
FRAME_FREE);
}
}
fn check_is_allocated(address: PhysicalAddress, length: usize) -> bool {
let allocator = FRAME_ALLOCATOR.lock();
assert!(allocator.initialized, "The frame allocator was not initialized");
(address.floor()..(address + length).ceil()).step_by(PAGE_SIZE).all(|frame| {
let frame_index = addr_to_frame(frame.addr());
allocator.memory_bitmap.get_bit(frame_index) == FRAME_OCCUPIED
})
}
fn check_is_reserved(address: PhysicalAddress, length: usize) -> bool {
Self::check_is_allocated(address, length)
}
}
impl FrameAllocatorTrait for FrameAllocator {
#[allow(clippy::match_bool)]
fn allocate_region(length: usize) -> Result<PhysicalMemRegion, KernelError> {
check_nonzero_length(length)?;
check_size_aligned(length, PAGE_SIZE)?;
let nr_frames = length / PAGE_SIZE;
let mut allocator = FRAME_ALLOCATOR.lock();
assert!(allocator.initialized, "The frame allocator was not initialized");
let mut start_index = 0usize;
while start_index + nr_frames <= allocator.memory_bitmap.bit_length() {
let mut temp_len = 0usize;
loop {
match allocator.memory_bitmap.get_bit(start_index + temp_len) {
FRAME_OCCUPIED => {
start_index += temp_len + 1;
break;
}
FRAME_FREE => {
temp_len += 1;
if temp_len == nr_frames {
allocator.memory_bitmap.set_bits_area(start_index..start_index+temp_len, FRAME_OCCUPIED);
let allocated = PhysicalMemRegion {
start_addr: frame_to_addr(start_index),
frames: nr_frames,
should_free_on_drop: true
};
debug!("Allocated physical region: {:?}", allocated);
return Ok(allocated);
}
}
}
}
}
info!("Failed physical allocation for {} consecutive frames", nr_frames);
Err(KernelError::PhysicalMemoryExhaustion { backtrace: Backtrace::new() })
}
fn allocate_frames_fragmented(length: usize) -> Result<Vec<PhysicalMemRegion>, KernelError> {
check_nonzero_length(length)?;
check_size_aligned(length, PAGE_SIZE)?;
let requested = length / PAGE_SIZE;
let mut allocator_lock = FRAME_ALLOCATOR.lock();
assert!(allocator_lock.initialized, "The frame allocator was not initialized");
let mut collected_frames = 0;
let mut collected_regions = Vec::new();
let mut current_hole = PhysicalMemRegion { start_addr: 0, frames: 0, should_free_on_drop: true };
while addr_to_frame(current_hole.start_addr) + (requested - collected_frames) <= allocator_lock.memory_bitmap.bit_length() {
while current_hole.frames < requested - collected_frames {
let considered_frame = addr_to_frame(current_hole.start_addr) + current_hole.frames;
if allocator_lock.memory_bitmap.get_bit(considered_frame) == FRAME_FREE {
allocator_lock.memory_bitmap.set_bit(considered_frame, FRAME_OCCUPIED);
current_hole.frames += 1;
} else {
break;
}
}
let cur_hole_addr = current_hole.start_addr;
let cur_hole_frames = current_hole.frames;
if current_hole.frames > 0 {
drop(allocator_lock);
collected_frames += current_hole.frames;
collected_regions.push(current_hole);
if collected_frames == requested {
debug!("Allocated physical regions: {:?}", collected_regions);
return Ok(collected_regions)
}
allocator_lock = FRAME_ALLOCATOR.lock();
}
current_hole = PhysicalMemRegion {
start_addr: match cur_hole_addr.checked_add((cur_hole_frames + 1) * PAGE_SIZE) {
Some(sum_addr) => sum_addr,
None => break
},
frames: 0,
should_free_on_drop: true
};
}
drop(allocator_lock);
info!("Failed physical allocation for {} non consecutive frames", requested);
Err(KernelError::PhysicalMemoryExhaustion { backtrace: Backtrace::new() })
}
}
#[cfg(not(test))]
pub fn init(boot_info: &BootInformation) {
let mut allocator = FRAME_ALLOCATOR.lock();
let memory_map_tag = boot_info.memory_map_tag()
.expect("GRUB, you're drunk. Give us our memory_map_tag.");
for memarea in memory_map_tag.memory_areas() {
if memarea.start_address() > u64::from(u32::max_value()) || memarea.end_address() > u64::from(u32::max_value()) {
continue;
}
if memarea.memory_type() == 1 {
mark_area_free(&mut allocator.memory_bitmap,
memarea.start_address() as usize,
memarea.end_address() as usize);
} else {
mark_area_reserved(&mut allocator.memory_bitmap,
memarea.start_address() as usize,
memarea.end_address() as usize);
}
}
drop(allocator);
get_kernel_memory().reserve_kernel_land_frames();
let mut allocator = FRAME_ALLOCATOR.lock();
for module in boot_info.module_tags() {
mark_area_reserved(&mut allocator.memory_bitmap,
module.start_address() as usize, module.end_address() as usize);
}
mark_area_reserved(&mut allocator.memory_bitmap,
0x00000000,
0x00000001);
if log_enabled!(::log::Level::Info) {
let mut cur = None;
for (i, bitmap) in allocator.memory_bitmap.iter().enumerate() {
for j in 0..8 {
let curaddr = (i * 8 + j) * crate::paging::PAGE_SIZE;
if bitmap & (1 << j) != 0 {
match cur {
None => cur = Some((FRAME_FREE, curaddr)),
Some((FRAME_OCCUPIED, last)) => {
info!("{:#010x} - {:#010x} OCCUPIED", last, curaddr);
cur = Some((FRAME_FREE, curaddr));
},
_ => ()
}
} else {
match cur {
None => cur = Some((FRAME_OCCUPIED, curaddr)),
Some((FRAME_FREE, last)) => {
info!("{:#010x} - {:#010x} AVAILABLE", last, curaddr);
cur = Some((FRAME_OCCUPIED, curaddr));
},
_ => ()
}
}
}
}
match cur {
Some((FRAME_FREE, last)) => info!("{:#010x} - {:#010x} AVAILABLE", last, 0xFFFFFFFFu32),
Some((FRAME_OCCUPIED, last)) => info!("{:#010x} - {:#010x} OCCUPIED", last, 0xFFFFFFFFu32),
_ => ()
}
}
allocator.initialized = true
}
#[cfg(test)]
pub use self::test::init;
fn mark_area_reserved(bitmap: &mut [u8],
start_addr: usize,
end_addr: usize) {
info!("Setting {:#010x}..{:#010x} to reserved", round_to_page(start_addr), round_to_page_upper(end_addr));
bitmap.set_bits_area(
addr_to_frame(round_to_page(start_addr))
..
addr_to_frame(round_to_page_upper(end_addr)),
FRAME_OCCUPIED);
}
fn mark_area_free(bitmap: &mut [u8],
start_addr: usize,
end_addr: usize) {
info!("Setting {:#010x}..{:#010x} to available", round_to_page(start_addr), round_to_page_upper(end_addr));
bitmap.set_bits_area(
addr_to_frame(round_to_page_upper(start_addr))
..
addr_to_frame(round_to_page(end_addr)),
FRAME_FREE);
}
pub fn mark_frame_bootstrap_allocated(addr: PhysicalAddress) {
debug!("Setting {:#010x} to boostrap allocked", addr.addr());
assert_eq!(addr.addr() & FRAME_OFFSET_MASK, 0x000);
let bit = addr_to_frame(addr.addr());
let mut allocator = FRAME_ALLOCATOR.lock();
if allocator.memory_bitmap.get_bit(bit) != FRAME_FREE {
panic!("Frame being marked reserved was already allocated");
}
allocator.memory_bitmap.set_bit(bit, FRAME_OCCUPIED);
}
#[cfg(test)]
mod test {
use super::*;
const ALL_MEMORY: usize = FRAMES_BITMAP_SIZE * 8 * PAGE_SIZE;
pub fn init() -> FrameAllocatorInitialized {
let mut allocator = FRAME_ALLOCATOR.lock();
assert_eq!(allocator.initialized, false, "frame_allocator::init() was called twice");
mark_area_free(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
mark_area_reserved(&mut allocator.memory_bitmap, PAGE_SIZE * 3, PAGE_SIZE * 3 + 1);
allocator.initialized = true;
FrameAllocatorInitialized(())
}
#[must_use]
pub struct FrameAllocatorInitialized(());
impl ::core::ops::Drop for FrameAllocatorInitialized {
fn drop(&mut self) { FRAME_ALLOCATOR.lock().initialized = false; }
}
#[test]
fn ok() {
let _f = crate::frame_allocator::init();
let a = FrameAllocator::allocate_frame().unwrap();
let b = FrameAllocator::allocate_region(2 * PAGE_SIZE).unwrap();
let c_vec = FrameAllocator::allocate_frames_fragmented(3 * PAGE_SIZE).unwrap();
drop(a);
drop(b);
drop(c_vec);
}
#[test]
fn fragmented() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_free(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
mark_area_reserved(&mut allocator.memory_bitmap, 2 * PAGE_SIZE, 7 * PAGE_SIZE);
drop(allocator);
let frames = FrameAllocator::allocate_frames_fragmented(5 * PAGE_SIZE).unwrap();
assert_eq!(frames.len(), 2);
assert_eq!(frames[0].address(), PhysicalAddress(0x00000000));
assert_eq!(frames[0].size(), 2 * PAGE_SIZE);
assert_eq!(frames[1].address(), PhysicalAddress(7 * PAGE_SIZE));
assert_eq!(frames[1].size(), 3 * PAGE_SIZE);
}
#[test]
fn zero() {
let _f = crate::frame_allocator::init();
FrameAllocator::allocate_region(0).unwrap_err();
FrameAllocator::allocate_frames_fragmented(0).unwrap_err();
}
#[test] #[should_panic] fn no_init_frame() { let _ = FrameAllocator::allocate_frame(); }
#[test] #[should_panic] fn no_init_region() { let _ = FrameAllocator::allocate_region(PAGE_SIZE); }
#[test] #[should_panic] fn no_init_fragmented() { let _ = FrameAllocator::allocate_frames_fragmented(PAGE_SIZE); }
#[test]
fn physical_oom_frame() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_reserved(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
drop(allocator);
match FrameAllocator::allocate_frame() {
Err(KernelError::PhysicalMemoryExhaustion { .. }) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
}
}
#[test]
fn physical_oom_frame_threshold() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_reserved(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
mark_area_free(&mut allocator.memory_bitmap, ALL_MEMORY - PAGE_SIZE, ALL_MEMORY);
drop(allocator);
FrameAllocator::allocate_frame().unwrap();
}
#[test]
fn physical_oom_region() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_reserved(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
mark_area_free(&mut allocator.memory_bitmap,
ALL_MEMORY - 3 * PAGE_SIZE,
ALL_MEMORY);
drop(allocator);
match FrameAllocator::allocate_region(4 * PAGE_SIZE) {
Err(KernelError::PhysicalMemoryExhaustion { .. }) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
}
}
#[test]
fn physical_oom_region_threshold() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_reserved(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
mark_area_free(&mut allocator.memory_bitmap,
ALL_MEMORY - 3 * PAGE_SIZE,
ALL_MEMORY);
drop(allocator);
FrameAllocator::allocate_region(3 * PAGE_SIZE).unwrap();
}
#[test]
fn physical_oom_fragmented() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_free(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
drop(allocator);
match FrameAllocator::allocate_frames_fragmented(ALL_MEMORY + PAGE_SIZE) {
Err(KernelError::PhysicalMemoryExhaustion { .. }) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
}
}
#[test]
fn physical_oom_threshold_fragmented() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_free(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
drop(allocator);
FrameAllocator::allocate_frames_fragmented(ALL_MEMORY).unwrap();
}
#[test]
fn allocate_last_frame() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_free(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
mark_area_reserved(&mut allocator.memory_bitmap, 0, ALL_MEMORY - PAGE_SIZE);
drop(allocator);
let frame = FrameAllocator::allocate_frame().unwrap();
drop(frame);
let frame = FrameAllocator::allocate_region(PAGE_SIZE).unwrap();
drop(frame);
let frame = FrameAllocator::allocate_frames_fragmented(PAGE_SIZE).unwrap();
drop(frame);
let frame = FrameAllocator::allocate_frame().unwrap();
match FrameAllocator::allocate_frame() {
Err(KernelError::PhysicalMemoryExhaustion {..} ) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
};
drop(frame);
}
#[test]
fn oom_hard() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_reserved(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
mark_area_free(&mut allocator.memory_bitmap, 2 * PAGE_SIZE, 3 * PAGE_SIZE);
drop(allocator);
match FrameAllocator::allocate_region(2 * PAGE_SIZE) {
Err(KernelError::PhysicalMemoryExhaustion { .. }) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
}
match FrameAllocator::allocate_frames_fragmented(2 * PAGE_SIZE) {
Err(KernelError::PhysicalMemoryExhaustion { .. }) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
}
let frame = FrameAllocator::allocate_frame().unwrap();
match FrameAllocator::allocate_frame() {
Err(KernelError::PhysicalMemoryExhaustion { .. }) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
}
drop(frame);
}
#[test]
fn physical_oom_doesnt_leak() {
let _f = crate::frame_allocator::init();
let mut allocator = FRAME_ALLOCATOR.lock();
mark_area_free(&mut allocator.memory_bitmap, 0, ALL_MEMORY);
drop(allocator);
let half_left = FrameAllocator::allocate_region(ALL_MEMORY / 2).unwrap();
let half_right = FrameAllocator::allocate_region(ALL_MEMORY / 2).unwrap();
match FrameAllocator::allocate_frame() {
Err(KernelError::PhysicalMemoryExhaustion {..} ) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
};
drop(half_left);
match FrameAllocator::allocate_frames_fragmented(ALL_MEMORY / 2 + PAGE_SIZE) {
Err(KernelError::PhysicalMemoryExhaustion {..} ) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
};
let half_left = FrameAllocator::allocate_frames_fragmented( ALL_MEMORY / 2).unwrap();
match FrameAllocator::allocate_frame() {
Err(KernelError::PhysicalMemoryExhaustion {..} ) => (),
unexpected_err => panic!("test failed: {:#?}", unexpected_err)
};
drop(half_left);
drop(half_right);
}
}