Merge all VM components #61

Closed
td1223 wants to merge 6 commits from vm/shared-ro-executables/gleb into vm/virtual-memory/themis-synch
8 changed files with 452 additions and 75 deletions

View File

@@ -33,6 +33,7 @@
#endif
#ifdef VM
#include "vm/frame.h"
#include "vm/page.h"
#include "devices/swap.h"
#endif
#ifdef FILESYS
@@ -104,6 +105,7 @@ main (void)
paging_init ();
#ifdef VM
frame_init ();
shared_file_pages_init ();
#endif
/* Segmentation. */

View File

@@ -265,11 +265,24 @@ thread_create (const char *name, int priority,
#ifdef USERPROG
/* Initialize the thread's file descriptor table. */
t->fd_counter = MINIMUM_USER_FD;
bool success = hash_init (&t->open_files, fd_hash, fd_less, NULL);
if (success)
{
success = hash_init (&t->child_results, process_result_hash,
process_result_less, t);
if (!success)
hash_destroy (&t->open_files, NULL);
#ifdef VM
else
{
success = init_pages (&t->pages);
if (!success)
hash_destroy (&t->child_results, NULL);
}
#endif
}
if (!hash_init (&t->open_files, fd_hash, fd_less, NULL)
|| !hash_init (&t->child_results, process_result_hash,
process_result_less, t)
|| !hash_init (&t->pages, page_hash, page_less, NULL))
if (!success)
{
palloc_free_page (t);
free (t->result);

View File

@@ -292,7 +292,8 @@ fetch_page (void *upage, bool write)
switch (page->type) {
case PAGE_MMAP:
case PAGE_FILE:
success = page_load_file (page, page->writable);
case PAGE_SHARED:
success = page_load_file (page);
if (success && page->type == PAGE_FILE)
{
hash_delete (&thread_current ()->pages, &page->elem);

View File

@@ -2,9 +2,12 @@
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include "devices/swap.h"
#include "threads/init.h"
#include "threads/pte.h"
#include "threads/palloc.h"
#include "vm/frame.h"
#include "vm/page.h"
static uint32_t *active_pd (void);
@@ -39,8 +42,14 @@ pagedir_destroy (uint32_t *pd)
uint32_t *pte;
for (pte = pt; pte < pt + PGSIZE / sizeof *pte; pte++)
if (*pte & PTE_P)
palloc_free_page (pte_get_page (*pte));
{
if (page_is_shared_pte (pte))
continue;
else if (page_in_swap_pte (pte))
swap_drop (page_get_swap_pte (pte));
else if (*pte & PTE_P)
frame_free (pte_get_page (*pte));
}
palloc_free_page (pt);
}
palloc_free_page (pd);

View File

@@ -35,7 +35,7 @@ struct frame_metadata
{
void *frame; /* The kernel virtual address holding the frame. */
void *upage; /* The user virtual address pointing to the frame. */
struct thread *owner; /* Pointer to the thread that owns the frame. */
struct list owners; /* List of threads that own the frame. */
struct hash_elem hash_elem; /* Tracks the position of the frame metadata
within 'frame_table', whose key is the
kernel virtual address of the frame. */
@@ -50,6 +50,8 @@ hash_less_func frame_metadata_less;
static struct list_elem *lru_next (struct list_elem *e);
static struct list_elem *lru_prev (struct list_elem *e);
static struct frame_metadata *get_victim (void);
static void free_owners (struct list *owners);
static struct frame_metadata *frame_metadata_find (void *frame);
/* Initialize the frame system by initializing the frame (hash) table with
the frame_metadata hashing and comparison functions, as well as initializing
@@ -92,7 +94,10 @@ frame_alloc (enum palloc_flags flags, void *upage, struct thread *owner)
ASSERT (victim != NULL); /* get_victim () should never return null. */
/* 2. Swap out victim into disk. */
page_insert_swapped (victim->upage, victim->frame, victim->owner);
page_insert_swapped (victim->upage, victim->frame, &victim->owners);
/* Free victim's owners. */
free_owners (&victim->owners);
/* If zero flag is set, zero out the victim page. */
if (flags & PAL_ZERO)
@@ -113,6 +118,8 @@ frame_alloc (enum palloc_flags flags, void *upage, struct thread *owner)
/* Must own lru_lock here, as otherwise there is a race condition
with next_victim either being NULL or uninitialized. */
frame_metadata = malloc (sizeof (struct frame_metadata));
if (frame_metadata == NULL)
PANIC ("Couldn't allocate memory for frame metadata!\n");
frame_metadata->frame = frame;
/* Newly allocated frames are pushed to the back of the circular queue
@@ -132,10 +139,14 @@ frame_alloc (enum palloc_flags flags, void *upage, struct thread *owner)
hash_insert (&frame_table, &frame_metadata->hash_elem);
}
struct frame_owner *frame_owner = malloc (sizeof (struct frame_owner));
if (frame_owner == NULL)
PANIC ("Couldn't allocate memory for frame owner!\n");
frame_owner->owner = owner;
list_init (&frame_metadata->owners);
list_push_back (&frame_metadata->owners, &frame_owner->elem);
frame_metadata->upage = upage;
frame_metadata->owner = owner;
lock_release (&lru_lock);
return frame_metadata->frame;
}
@@ -145,18 +156,15 @@ frame_alloc (enum palloc_flags flags, void *upage, struct thread *owner)
void
frame_free (void *frame)
{
struct frame_metadata key_metadata;
key_metadata.frame = frame;
struct frame_metadata *frame_metadata = frame_metadata_find (frame);
if (frame_metadata == NULL)
PANIC ("Attempted to free a frame at kernel address %p, "
"but this address is not allocated!\n",
frame);
struct hash_elem *e =
hash_delete (&frame_table, &key_metadata.hash_elem);
if (e == NULL) PANIC ("Attempted to free a frame at kernel address %p, "
"but this address is not allocated!\n", frame);
struct frame_metadata *frame_metadata =
hash_entry (e, struct frame_metadata, hash_elem);
free_owners (&frame_metadata->owners);
lock_acquire (&lru_lock);
hash_delete (&frame_table, &frame_metadata->hash_elem);
list_remove (&frame_metadata->list_elem);
/* If we're freeing the frame marked as the next victim, update
@@ -175,33 +183,113 @@ frame_free (void *frame)
palloc_free_page (frame);
}
/* Add a thread to a frame's frame_metadata owners list. */
bool
frame_owner_insert (void *frame, struct thread *owner)
{
struct frame_metadata *frame_metadata = frame_metadata_find (frame);
if (frame_metadata == NULL)
return false;
struct frame_owner *frame_owner = malloc (sizeof (struct frame_owner));
if (frame_owner == NULL)
return false;
frame_owner->owner = owner;
list_push_back (&frame_metadata->owners, &frame_owner->elem);
return true;
}
/* Remove and deallocate a frame owner from the frame_metadata owners list.
*/
void
frame_owner_remove (void *frame, struct thread *owner)
{
struct frame_metadata *frame_metadata = frame_metadata_find (frame);
if (frame_metadata == NULL)
PANIC ("Attempted to remove an owner from a frame at kernel "
"address %p, but this address is not allocated!\n",
frame);
struct list_elem *oe;
for (oe = list_begin (&frame_metadata->owners);
oe != list_end (&frame_metadata->owners);)
{
struct frame_owner *frame_owner
= list_entry (oe, struct frame_owner, elem);
oe = list_next (oe);
if (frame_owner->owner == owner)
{
list_remove (&frame_owner->elem);
free (frame_owner);
return;
}
}
NOT_REACHED ();
}
/* Find a frame_metadata entry in the frame table. */
static struct frame_metadata *
frame_metadata_find (void *frame)
{
struct frame_metadata key_metadata;
key_metadata.frame = frame;
struct hash_elem *e = hash_find (&frame_table, &key_metadata.hash_elem);
if (e == NULL)
return NULL;
return hash_entry (e, struct frame_metadata, hash_elem);
}
/* TODO: Account for page aliases when checking accessed bit. */
/* A pre-condition for calling this function is that the calling thread
owns lru_lock and that lru_list is non-empty. */
static struct frame_metadata *
get_victim (void)
{
struct list_elem *e = next_victim;
struct list_elem *ve = next_victim;
struct frame_metadata *frame_metadata;
uint32_t *pd;
void *upage;
for (;;)
bool found = false;
while (!found)
{
frame_metadata = list_entry (e, struct frame_metadata, list_elem);
pd = frame_metadata->owner->pagedir;
upage = frame_metadata->upage;
e = lru_next (e);
frame_metadata = list_entry (ve, struct frame_metadata, list_elem);
ve = lru_next (ve);
struct list_elem *oe;
if (!pagedir_is_accessed (pd, upage))
break;
/* Returns once a frame that was not accessed by any owner is found. */
found = true;
for (oe = list_begin (&frame_metadata->owners);
oe != list_end (&frame_metadata->owners); oe = list_next (oe))
{
struct frame_owner *frame_owner
= list_entry (oe, struct frame_owner, elem);
uint32_t *pd = frame_owner->owner->pagedir;
void *upage = frame_metadata->upage;
pagedir_set_accessed (pd, upage, false);
if (pagedir_is_accessed (pd, upage))
{
found = false;
pagedir_set_accessed (pd, upage, false);
}
}
}
next_victim = e;
next_victim = ve;
return frame_metadata;
}
static void
free_owners (struct list *owners)
{
struct list_elem *oe;
for (oe = list_begin (owners); oe != list_end (owners);)
{
struct frame_owner *frame_owner
= list_entry (oe, struct frame_owner, elem);
oe = list_remove (oe);
free (frame_owner);
}
}
/* Hash function for frame metadata, used for storing entries in the
frame table. */
unsigned

View File

@@ -4,8 +4,17 @@
#include "threads/thread.h"
#include "threads/palloc.h"
struct frame_owner
{
struct thread *owner; /* The thread that owns the frame. */
struct list_elem elem; /* List element for the list of owners. */
};
void frame_init (void);
void *frame_alloc (enum palloc_flags, void *, struct thread *);
void frame_free (void *frame);
bool frame_owner_insert (void *frame, struct thread *owner);
void frame_owner_remove (void *frame, struct thread *owner);
#endif /* vm/frame.h */

View File

@@ -6,18 +6,44 @@
#include "threads/pte.h"
#include "threads/malloc.h"
#include "threads/palloc.h"
#include "threads/synch.h"
#include "devices/swap.h"
#include "userprog/process.h"
#include "userprog/pagedir.h"
#include "vm/frame.h"
#define SWAP_FLAG_BIT 9
#define SHARED_FLAG_BIT 10
#define ADDR_START_BIT 12
struct hash shared_file_pages;
struct lock shared_file_pages_lock;
static unsigned page_hash (const struct hash_elem *e, void *aux UNUSED);
static bool page_less (const struct hash_elem *a_, const struct hash_elem *b_,
void *aux UNUSED);
static void page_flag_shared (struct thread *owner, void *upage, bool shared);
static unsigned shared_file_page_hash (const struct hash_elem *e,
void *aux UNUSED);
static bool shared_file_page_less (const struct hash_elem *a_,
const struct hash_elem *b_,
void *aux UNUSED);
static struct shared_file_page *shared_file_page_get (struct file *file,
void *upage);
/* Initialise a supplementary page table. */
bool
init_pages (struct hash *pages)
{
ASSERT (pages != NULL);
return hash_init (pages, page_hash, page_less, NULL);
}
/* Hashing function needed for the SPT table. Returns a hash for an entry,
based on its upage. */
unsigned
page_hash (const struct hash_elem *e, UNUSED void *aux)
static unsigned
page_hash (const struct hash_elem *e, void *aux UNUSED)
{
struct page_entry *page = hash_entry (e, struct page_entry, elem);
return hash_ptr (page->upage);
@@ -25,7 +51,7 @@ page_hash (const struct hash_elem *e, UNUSED void *aux)
/* Comparator function for the SPT table. Compares two entries based on their
upages. */
bool
static bool
page_less (const struct hash_elem *a_, const struct hash_elem *b_,
void *aux UNUSED)
{
@@ -42,38 +68,61 @@ static void page_set_swap (struct thread *owner, uint32_t *pte,
/* Swap out 'owner' process's 'upage' stored at 'kpage'. Then, allocate and
insert a new page entry into the user process thread's SPT representing
this swapped out page. */
struct page_entry *
page_insert_swapped (void *upage, void *kpage, struct thread *owner)
bool
page_insert_swapped (void *upage, void *kpage, struct list *owners)
{
struct file *exec_file = NULL;
struct list_elem *e;
for (e = list_begin (owners); e != list_end (owners); e = list_next (e))
{
struct thread *owner = list_entry (e, struct frame_owner, elem)->owner;
uint32_t *pte = lookup_page (owner->pagedir, upage, false);
if (exec_file != NULL || page_is_shared_pte (pte))
{
ASSERT (page_is_shared_pte (pte));
pagedir_clear_page (owner->pagedir, upage);
exec_file = owner->exec_file;
ASSERT (exec_file != NULL);
continue;
}
ASSERT (list_size (owners) == 1);
/* 1. Initialize swapped page entry. */
struct page_entry *page = malloc(sizeof (struct page_entry));
if (page == NULL)
return NULL;
/* 1. Initialize swapped page entry. */
struct page_entry *page = malloc (sizeof (struct page_entry));
if (page == NULL)
return false;
page->upage = upage;
lock_init (&page->lock);
page->upage = upage;
lock_init (&page->lock);
/* Mark page as 'swapped' and flag the page directory as having
been modified *before* eviction begins to prevent the owner of the
victim page from accessing/modifying it mid-eviction. */
/* TODO: We need to stop the process from destroying pagedir mid-eviction,
as this could render the page table entry invalid. */
uint32_t *pte = lookup_page (owner->pagedir, upage, false);
ASSERT (pte != NULL);
/* Mark page as 'swapped' and flag the page directory as having
been modified *before* eviction begins to prevent the owner of the
victim page from accessing/modifying it mid-eviction. */
/* TODO: We need to stop the process from destroying pagedir
mid-eviction, as this could render the page table entry invalid. */
ASSERT (pte != NULL);
page_flag_swap (pte, true);
lock_acquire (&page->lock);
struct hash_elem *elem = hash_insert (&owner->pages, &page->elem);
ASSERT(elem == NULL);
page_flag_swap (pte, true);
lock_acquire (&page->lock);
struct hash_elem *elem = hash_insert (&owner->pages, &page->elem);
ASSERT (elem == NULL);
pagedir_clear_page (owner->pagedir, upage);
pagedir_clear_page (owner->pagedir, upage);
size_t swap_slot = swap_out (kpage);
page_set_swap (owner, pte, swap_slot);
size_t swap_slot = swap_out (kpage);
page_set_swap (owner, pte, swap_slot);
lock_release (&page->lock);
return page;
lock_release (&page->lock);
}
if (exec_file != NULL)
{
lock_acquire (&shared_file_pages_lock);
struct shared_file_page *sfp = shared_file_page_get (exec_file, upage);
sfp->frame = NULL;
sfp->swap_slot = swap_out (kpage);
lock_release (&shared_file_pages_lock);
}
return true;
}
/* Allocate and insert a new page entry into the user process thread's
@@ -83,6 +132,16 @@ page_insert_file (struct file *file, off_t ofs, void *upage,
uint32_t read_bytes, uint32_t zero_bytes, bool writable,
enum page_type type)
{
/* If page exists, just update it. */
struct page_entry *existing = page_get (upage);
if (existing != NULL)
{
ASSERT (existing->read_bytes == read_bytes);
ASSERT (existing->zero_bytes == zero_bytes);
existing->writable = existing->writable || writable;
return existing;
}
struct page_entry *page = malloc(sizeof (struct page_entry));
if (page == NULL)
return NULL;
@@ -116,20 +175,71 @@ page_get (void *upage)
}
bool
page_load_file (struct page_entry *page, bool writable)
page_load_file (struct page_entry *page)
{
/* Allocate a frame for the page. If a frame allocation fails, then
frame_alloc should try to evict a page. If it is still NULL, the OS
panics as this should not happen if eviction is working correctly. */
struct thread *t = thread_current ();
bool shareable = !page->writable && file_compare (page->file, t->exec_file);
if (shareable)
{
lock_acquire (&shared_file_pages_lock);
struct shared_file_page *sfp
= shared_file_page_get (page->file, page->upage);
if (sfp != NULL)
{
/* Frame exists, just install it. */
if (sfp->frame != NULL)
{
if (!install_page (page->upage, sfp->frame, page->writable))
{
lock_release (&shared_file_pages_lock);
return false;
}
/* First time adding the shared page, so add thread as owner. */
if (page->type != PAGE_SHARED)
{
frame_owner_insert (sfp->frame, t);
}
}
/* Shared page is in swap. Load it. */
else
{
void *frame = frame_alloc (PAL_USER, page->upage, t);
if (frame == NULL)
PANIC (
"Could not allocate a frame to load page into memory.");
swap_in (frame, sfp->swap_slot);
if (!install_page (page->upage, frame, false))
{
frame_free (frame);
lock_release (&shared_file_pages_lock);
return false;
}
}
page_flag_shared (t, page->upage, true);
if (page->type != PAGE_SHARED)
{
sfp->ref_count++;
page->type = PAGE_SHARED;
}
lock_release (&shared_file_pages_lock);
return true;
}
}
void *frame = frame_alloc (PAL_USER, page->upage, t);
pagedir_set_accessed (t->pagedir, page->upage, true);
if (frame == NULL)
PANIC ("Could not allocate a frame to load page into memory.");
/* Map the page to the frame. */
if (!install_page (page->upage, frame, writable))
if (!install_page (page->upage, frame, page->writable))
{
if (shareable)
lock_release (&shared_file_pages_lock);
frame_free (frame);
return false;
}
@@ -140,6 +250,8 @@ page_load_file (struct page_entry *page, bool writable)
file_seek (page->file, page->offset);
if (file_read (page->file, frame, page->read_bytes) != (int) page->read_bytes)
{
if (shareable)
lock_release (&shared_file_pages_lock);
frame_free (frame);
return false;
}
@@ -147,6 +259,27 @@ page_load_file (struct page_entry *page, bool writable)
/* Zero out the remaining bytes in the frame. */
memset (frame + page->read_bytes, 0, page->zero_bytes);
/* If file page is read-only, make it shared. */
if (shareable)
{
struct shared_file_page *sfp = malloc (sizeof (struct shared_file_page));
if (sfp == NULL)
{
lock_release (&shared_file_pages_lock);
frame_free (frame);
return false;
}
sfp->file = page->file;
sfp->upage = page->upage;
sfp->frame = frame;
sfp->swap_slot = 0;
sfp->ref_count = 1;
hash_insert (&shared_file_pages, &sfp->elem);
page_flag_shared (t, page->upage, true);
page->type = PAGE_SHARED;
lock_release (&shared_file_pages_lock);
}
/* Mark the page as loaded successfully. */
return true;
}
@@ -156,7 +289,28 @@ page_load_file (struct page_entry *page, bool writable)
void
page_cleanup (struct hash_elem *e, void *aux UNUSED)
{
free (hash_entry (e, struct page_entry, elem));
struct page_entry *page = hash_entry (e, struct page_entry, elem);
if (page->type == PAGE_SHARED)
{
lock_acquire (&shared_file_pages_lock);
struct shared_file_page *sfp
= shared_file_page_get (page->file, page->upage);
ASSERT (sfp != NULL);
if (sfp->frame != NULL)
frame_owner_remove (sfp->frame, thread_current ());
sfp->ref_count--;
if (sfp->ref_count == 0)
{
hash_delete (&shared_file_pages, &sfp->elem);
if (sfp->frame != NULL)
frame_free (sfp->frame);
else
swap_drop (sfp->swap_slot);
free (sfp);
}
lock_release (&shared_file_pages_lock);
}
free (page);
}
/* Flags the provided page table entry as representing a swapped out page. */
@@ -189,8 +343,14 @@ bool
page_in_swap (struct thread *owner, void *upage)
{
uint32_t *pte = lookup_page (owner->pagedir, upage, false);
return pte != NULL &&
(*pte & (1 << SWAP_FLAG_BIT)) != 0;
return page_in_swap_pte (pte);
}
/* Returns true iff the page table entry is marked to be in the swap disk. */
bool
page_in_swap_pte (uint32_t *pte)
{
return pte != NULL && (*pte & (1 << SWAP_FLAG_BIT)) != 0;
}
/* Given that the page with user address 'upage' owned by 'owner' is flagged
@@ -208,3 +368,82 @@ page_get_swap (struct thread *owner, void *upage)
page_flag_swap (pte, false);
return ((*pte & PTE_ADDR) >> ADDR_START_BIT);
}
/* Returns the swap slot stored in a PTE. */
size_t
page_get_swap_pte (uint32_t *pte)
{
ASSERT (pte != NULL);
ASSERT ((*pte & PTE_P) == 0);
return ((*pte & PTE_ADDR) >> ADDR_START_BIT);
}
/* Flags the provided page table entry as representing a shared page. */
static void
page_flag_shared (struct thread *owner, void *upage, bool shared)
{
uint32_t *pte = lookup_page (owner->pagedir, upage, false);
ASSERT (pte != NULL);
if (shared)
*pte |= (1 << SHARED_FLAG_BIT);
else
*pte &= ~(1 << SHARED_FLAG_BIT);
}
/* Returns true iff the page table entry is marked to be shared. */
bool
page_is_shared_pte (uint32_t *pte)
{
return pte != NULL && (*pte & (1 << SHARED_FLAG_BIT)) != 0;
}
/* Initializes the shared file pages hash table. */
void
shared_file_pages_init ()
{
if (!hash_init (&shared_file_pages, shared_file_page_hash,
shared_file_page_less, NULL))
PANIC ("Failed to initialize shared file pages hash table.");
lock_init (&shared_file_pages_lock);
}
/* Hash function for shared file pages, used for storing entries in the
shared file pages table. */
static unsigned
shared_file_page_hash (const struct hash_elem *e, void *aux UNUSED)
{
struct shared_file_page *sfp = hash_entry (e, struct shared_file_page, elem);
void *inode = file_get_inode (sfp->file);
void *upage = sfp->upage;
void *bytes[2] = { inode, upage };
return hash_bytes (bytes, sizeof (bytes));
}
/* 'less_func' comparison function for shared file pages, used for comparing
the keys of the shared file pages table. */
static bool
shared_file_page_less (const struct hash_elem *a_, const struct hash_elem *b_,
void *aux UNUSED)
{
const struct shared_file_page *a
= hash_entry (a_, struct shared_file_page, elem);
const struct shared_file_page *b
= hash_entry (b_, struct shared_file_page, elem);
return !file_compare (a->file, b->file) || a->upage < b->upage;
}
static struct shared_file_page *
shared_file_page_get (struct file *file, void *upage)
{
struct shared_file_page fake_sfp;
fake_sfp.file = file;
fake_sfp.upage = upage;
struct hash_elem *e = hash_find (&shared_file_pages, &fake_sfp.elem);
if (e == NULL)
return NULL;
return hash_entry (e, struct shared_file_page, elem);
}

View File

@@ -5,13 +5,16 @@
#include "threads/synch.h"
#include "filesys/off_t.h"
enum page_type {
enum page_type
{
PAGE_FILE,
PAGE_MMAP,
PAGE_EMPTY
PAGE_EMPTY,
PAGE_SHARED
};
struct page_entry {
struct page_entry
{
enum page_type type; /* Type of Data that should go into the page */
void *upage; /* Start Address of the User Page (Key of hash table). */
@@ -30,19 +33,32 @@ struct page_entry {
struct hash_elem elem; /* An elem for the hash table. */
};
unsigned page_hash (const struct hash_elem *e, void *aux);
bool page_less (const struct hash_elem *a_, const struct hash_elem *b_,
void *aux);
struct page_entry *page_insert_swapped (void *upage, void* kpage,
struct thread *owner);
struct shared_file_page
{
struct file *file;
void *upage;
void *frame;
size_t swap_slot;
int ref_count;
struct hash_elem elem;
};
bool init_pages (struct hash *pages);
bool page_insert_swapped (void *upage, void *kpage, struct list *owners);
struct page_entry *page_insert_file (struct file *file, off_t ofs, void *upage,
uint32_t read_bytes, uint32_t zero_bytes,
bool writable, enum page_type);
struct page_entry *page_get (void *upage);
bool page_load_file (struct page_entry *page, bool writable);
bool page_load_file (struct page_entry *page);
void page_cleanup (struct hash_elem *e, void *aux);
bool page_in_swap (struct thread *, void *);
bool page_in_swap_pte (uint32_t *pte);
size_t page_get_swap (struct thread *owner, void *upage);
size_t page_get_swap_pte (uint32_t *pte);
#endif /* vm/frame.h */
bool page_is_shared_pte (uint32_t *pte);
void shared_file_pages_init (void);
#endif