feat: set accessed bit to allocated frames in page_load and get_usr_kpage

.gitlab-ci.yml

@@ -37,4 +37,4 @@ test_vm:
   extends: .pintos_tests
   variables:
     DIR: vm
-    IGNORE: (tests/vm/pt-overflowstk|tests/vm/page-linear|tests/vm/page-parallel|tests/vm/page-merge-seq|tests/vm/page-merge-par|tests/vm/page-merge-stk|tests/vm/page-merge-mm|tests/vm/mmap-read|tests/vm/mmap-close|tests/vm/mmap-overlap|tests/vm/mmap-twice|tests/vm/mmap-write|tests/vm/mmap-exit|tests/vm/mmap-shuffle|tests/vm/mmap-clean|tests/vm/mmap-inherit|tests/vm/mmap-misalign|tests/vm/mmap-null|tests/vm/mmap-over-code|tests/vm/mmap-over-data|tests/vm/mmap-over-stk|tests/vm/mmap-remove)
+    IGNORE: (tests/vm/page-parallel|tests/vm/page-merge-seq|tests/vm/page-merge-par|tests/vm/page-merge-stk|tests/vm/page-merge-mm|tests/vm/mmap-read|tests/vm/mmap-close|tests/vm/mmap-overlap|tests/vm/mmap-twice|tests/vm/mmap-write|tests/vm/mmap-exit|tests/vm/mmap-shuffle|tests/vm/mmap-clean|tests/vm/mmap-inherit|tests/vm/mmap-misalign|tests/vm/mmap-null|tests/vm/mmap-over-code|tests/vm/mmap-over-data|tests/vm/mmap-over-stk|tests/vm/mmap-remove)

src/threads/thread.c

@@ -15,6 +15,7 @@
 #include "threads/switch.h"
 #include "threads/synch.h"
 #include "threads/vaddr.h"
+#include "vm/page.h"
 #ifdef USERPROG
 #include "userprog/process.h"
 #include "userprog/syscall.h"
@@ -262,9 +263,14 @@ thread_create (const char *name, int priority,
   /* Initialize the thread's file descriptor table. */
   t->fd_counter = MINIMUM_USER_FD;
 
-  if (!hash_init (&t->open_files, fd_hash, fd_less, NULL)
-      || !hash_init (&t->child_results, process_result_hash,
-                     process_result_less, t))
+  bool success = hash_init (&t->open_files, fd_hash, fd_less, NULL);
+  success = success && hash_init (&t->child_results, process_result_hash,
+                                  process_result_less, t);
+#ifdef VM
+  success = success && hash_init (&t->pages, page_hash, page_less, NULL);
+#endif
+
+  if (!success)
     {
       palloc_free_page (t);
       free (t->result);

src/threads/thread.h

@@ -143,6 +143,10 @@ struct thread
     struct hash open_files;             /* Hash Table of FD -> Struct File. */
 #endif
 
+#ifdef VM
+  struct hash pages;                    /* Table of open user pages. */
+#endif
+
     void *curr_esp;
 
     /* Owned by thread.c. */

src/userprog/exception.c

@@ -1,16 +1,25 @@
 #include "userprog/exception.h"
 #include <inttypes.h>
 #include <stdio.h>
+#include "stdbool.h"
 #include "userprog/gdt.h"
 #include "threads/interrupt.h"
 #include "threads/thread.h"
+#ifdef VM
 #include "vm/stackgrowth.h"
+#include "vm/frame.h"
+#include "vm/page.h"
+#include "devices/swap.h"
+#include "threads/vaddr.h"
+#include "userprog/pagedir.h"
+#endif
 
 /* Number of page faults processed. */
 static long long page_fault_cnt;
 
 static void kill (struct intr_frame *);
 static void page_fault (struct intr_frame *);
+bool try_fetch_page (void *upage, bool write);
 
 /* Registers handlers for interrupts that can be caused by user
    programs.
@@ -147,11 +156,41 @@ page_fault (struct intr_frame *f)
   user = (f->error_code & PF_U) != 0;
 
 #ifdef VM
-  if (user && not_present)
+  void *upage = pg_round_down (fault_addr);
+  if (not_present && is_user_vaddr(upage))
     {
-      if (handle_stack_fault (fault_addr, f->esp)) return;
+      struct thread *t = thread_current ();
+      void *esp = user ? f->esp : t->curr_esp;
+
+      /* Check if the non-present user page is in the swap partition.
+         If so, swap it back into main memory, updating the PTE for
+         the faulted virtual address to point to the newly allocated
+         frame. */
+      if (page_in_swap (t, fault_addr))
+        {
+          size_t swap_slot = page_get_swap (t, fault_addr);
+          void *kpage = frame_alloc (0, upage, t);
+          swap_in (kpage, swap_slot);
+
+          bool writeable = pagedir_is_writable (t->pagedir, upage);
+          if (pagedir_set_page (t->pagedir, upage, kpage, writeable)) return;
+        }
+
+      /* Handle user page faults that need to be resolved by dynamic
+         stack growth by checking if this is such a fault and responding
+         accordingly. */
+      if (handle_stack_fault (fault_addr, esp)) return;
+
+      /* Handle user page faults that need to be resolved by lazy loading
+         of executable files by checking if they contain entries in the
+         SPT hash map and responding accordingly. */
+      if (try_fetch_page (upage, write))
+        return;
     }
-  else
+
+  /* Allows for page faults within a kernel context to communicate with
+     user pages for sending error codes. */
+  if (!user)
     {
       f->eip = (void *)f->eax;
       f->eax = 0xffffffff;
@@ -170,3 +209,35 @@ page_fault (struct intr_frame *f)
   kill (f);
 }
 
+#ifdef VM
+bool
+try_fetch_page (void *upage, bool write)
+{
+  /* Check if the page is in the supplemental page table. That is, it is a page
+     that is expected to be in memory. */
+  struct page_entry *page = page_get (upage);
+  if (page == NULL)
+    return false;
+
+  /* An attempt to write to a non-writeable should fail. */
+  if (write && !page->writable)
+    return false;
+
+  /* Load the page into memory based on the type of data it is expecting. */
+  bool success = false;
+  switch (page->type) {
+    case PAGE_EXECUTABLE:
+      success = page_load (page, page->writable);
+      break;
+    default:
+      return false;
+  }
+
+  if (success && page->writable &&
+      !pagedir_is_writable(thread_current()->pagedir, upage))
+    pagedir_set_writable(thread_current()->pagedir, upage, true);
+
+  return success;
+}
+#endif
+

src/userprog/exception.h

@@ -1,6 +1,8 @@
 #ifndef USERPROG_EXCEPTION_H
 #define USERPROG_EXCEPTION_H
 
+#include <stdbool.h>
+
 /* Page fault error code bits that describe the cause of the exception.  */
 #define PF_P 0x1    /* 0: not-present page. 1: access rights violation. */
 #define PF_W 0x2    /* 0: read, 1: write. */
@@ -8,5 +10,7 @@
 
 void exception_init (void);
 void exception_print_stats (void);
+bool
+try_fetch_page (void *upage, bool write);
 
 #endif /* userprog/exception.h */

src/userprog/pagedir.c

@@ -7,7 +7,6 @@
 #include "threads/palloc.h"
 
 static uint32_t *active_pd (void);
-static void invalidate_pagedir (uint32_t *);
 
 /* Creates a new page directory that has mappings for kernel
    virtual addresses, but none for user virtual addresses.
@@ -278,7 +277,7 @@ active_pd (void)
    This function invalidates the TLB if PD is the active page
    directory.  (If PD is not active then its entries are not in
    the TLB, so there is no need to invalidate anything.) */
-static void
+void
 invalidate_pagedir (uint32_t *pd) 
 {
   if (active_pd () == pd) 

src/userprog/pagedir.h

@@ -17,5 +17,6 @@ void pagedir_set_accessed (uint32_t *pd, const void *upage, bool accessed);
 bool pagedir_is_writable (uint32_t *pd, const void *upage);
 void pagedir_set_writable (uint32_t *pd, const void *upage, bool writable);
 void pagedir_activate (uint32_t *pd);
+void invalidate_pagedir (uint32_t *pd);
 
 #endif /* userprog/pagedir.h */

src/userprog/process.c

@@ -24,6 +24,7 @@
 #include "threads/vaddr.h"
 #include "threads/synch.h"
 #include "devices/timer.h"
+#include "vm/page.h"
 #ifdef VM
 #include "vm/frame.h"
 #endif
@@ -118,7 +119,7 @@ process_execute (const char *cmd)
 
 static void *get_usr_kpage (enum palloc_flags flags, void *upage);
 static void free_usr_kpage (void *kpage);
-static bool install_page (void *upage, void *kpage, bool writable);
+bool install_page (void *upage, void *kpage, bool writable);
 
 static bool process_init_stack (char *cmd_saveptr, void **esp, char *file_name);
 static void *push_to_stack (void **esp, void *data, size_t data_size);
@@ -364,6 +365,9 @@ process_exit (void)
 
   /* Clean up all open files */
   hash_destroy (&cur->open_files, fd_cleanup);
+#ifdef VM
+  hash_destroy (&cur->pages, page_cleanup);
+#endif
 
   /* Close the executable file, implicitly allowing it to be written to. */
   if (cur->exec_file != NULL)
@@ -621,7 +625,9 @@ load (const char *file_name, void (**eip) (void), void **esp)
 
  done:
   /* We arrive here whether the load is successful or not. */
+#ifndef VM
   file_close (file);
+#endif
   lock_release (&filesys_lock);
   return success;
 }
@@ -689,12 +695,34 @@ validate_segment (const struct Elf32_Phdr *phdr, struct file *file)
    or disk read error occurs. */
 static bool
 load_segment (struct file *file, off_t ofs, uint8_t *upage,
-              uint32_t read_bytes, uint32_t zero_bytes, bool writable) 
+              uint32_t read_bytes, uint32_t zero_bytes, bool writable)
 {
   ASSERT ((read_bytes + zero_bytes) % PGSIZE == 0);
   ASSERT (pg_ofs (upage) == 0);
   ASSERT (ofs % PGSIZE == 0);
 
+#ifdef VM
+  while (read_bytes > 0 || zero_bytes > 0)
+    {
+      /* Calculate how to fill this page.
+         We will read PAGE_READ_BYTES bytes from FILE
+         and zero the final PAGE_ZERO_BYTES bytes. */
+      size_t page_read_bytes = read_bytes < PGSIZE ? read_bytes : PGSIZE;
+      size_t page_zero_bytes = PGSIZE - page_read_bytes;
+
+      /* Add the page metadata to the SPT to be lazy loaded later on */
+      if (page_insert (file, ofs, upage, page_read_bytes, page_zero_bytes,
+                       writable, PAGE_EXECUTABLE) == NULL)
+        return false;
+
+      /* Advance. */
+      read_bytes -= page_read_bytes;
+      zero_bytes -= page_zero_bytes;
+      ofs += PGSIZE;
+      upage += PGSIZE;
+    }
+  return true;
+#else
   file_seek (file, ofs);
   while (read_bytes > 0 || zero_bytes > 0) 
     {
@@ -744,6 +772,7 @@ load_segment (struct file *file, off_t ofs, uint8_t *upage,
       upage += PGSIZE;
     }
   return true;
+#endif
 }
 
 /* Create a minimal stack by mapping a zeroed page at the top of
@@ -782,6 +811,7 @@ get_usr_kpage (enum palloc_flags flags, void *upage)
     return NULL;
   else
     page = frame_alloc (flags, upage, t);
+  pagedir_set_accessed (t->pagedir, upage, true);
 #else
   page = palloc_get_page (flags | PAL_USER);
 #endif
@@ -809,7 +839,7 @@ free_usr_kpage (void *kpage)
    with palloc_get_page().
    Returns true on success, false if UPAGE is already mapped or
    if memory allocation fails. */
-static bool
+bool
 install_page (void *upage, void *kpage, bool writable)
 {
   struct thread *t = thread_current ();

src/userprog/process.h

@@ -8,4 +8,6 @@ int process_wait (tid_t);
 void process_exit (void);
 void process_activate (void);
 
+bool install_page (void *upage, void *kpage, bool writable);
+
 #endif /* userprog/process.h */

src/userprog/syscall.c

@@ -536,4 +536,4 @@ put_user (uint8_t *udst, uint8_t byte)
        : "=&a"(error_code), "=m"(*udst)
        : "q"(byte));
   return error_code != -1;
-}
+}

src/vm/frame.c

@@ -93,6 +93,13 @@ frame_alloc (enum palloc_flags flags, void *upage, struct thread *owner)
       ASSERT (victim != NULL); /* get_victim () should never return null. */
 
       /* 2. Swap out victim into disk. */
+      /* Mark page as 'not present' and flag the page directory as having
+         been modified *before* eviction begins to prevent the owner of the
+         victim page from accessing/modifying it mid-eviction. */
+      pagedir_clear_page (victim->owner->pagedir, victim->upage);
+
+      // TODO: Lock PTE of victim page for victim process.
+
       size_t swap_slot = swap_out (victim->frame);
       page_set_swap (victim->owner, victim->upage, swap_slot);
 
@@ -253,4 +260,3 @@ lru_prev (struct list_elem *e)
 
   return list_prev (e);
 }
-

src/vm/page.c

@@ -1,27 +1,145 @@
 #include "page.h"
-#include "userprog/pagedir.h"
+#include <string.h>
+#include <stdio.h>
+#include "filesys/file.h"
 #include "threads/pte.h"
+#include "threads/malloc.h"
+#include "threads/palloc.h"
+#include "userprog/process.h"
+#include "userprog/pagedir.h"
+#include "vm/frame.h"
 
 #define SWAP_FLAG_BIT 9
 #define ADDR_START_BIT 12
 
+/* Hashing function needed for the SPT table. Returns a hash for an entry,
+   based on its upage. */
+unsigned
+page_hash (const struct hash_elem *e, UNUSED void *aux)
+{
+  struct page_entry *page = hash_entry (e, struct page_entry, elem);
+  return hash_ptr (page->upage);
+}
+
+/* Comparator function for the SPT table. Compares two entries based on their
+   upages. */
+bool
+page_less (const struct hash_elem *a_, const struct hash_elem *b_,
+           void *aux UNUSED)
+{
+  const struct page_entry *a = hash_entry (a_, struct page_entry, elem);
+  const struct page_entry *b = hash_entry (b_, struct page_entry, elem);
+
+  return a->upage < b->upage;
+}
+
+/* Allocate and insert a new page entry into the thread's page table. */
+struct page_entry *
+page_insert (struct file *file, off_t ofs, void *upage, uint32_t read_bytes,
+             uint32_t zero_bytes, bool writable, enum page_type type)
+{
+  struct page_entry *page = malloc(sizeof (struct page_entry));
+  if (page == NULL)
+    return NULL;
+
+  page->file = file;
+  page->offset = ofs;
+  page->upage = upage;
+  page->read_bytes = read_bytes;
+  page->zero_bytes = zero_bytes;
+  page->writable = writable;
+  page->type = type;
+
+  hash_insert (&thread_current ()->pages, &page->elem);
+  return page;
+}
+
+/* Gets a page_entry from the starting address of the page. Returns NULL if no
+   such page_entry exists in the hash map.*/
+struct page_entry *
+page_get (void *upage)
+{
+  struct page_entry fake_page_entry;
+  fake_page_entry.upage = upage;
+
+  struct hash_elem *e
+    = hash_find (&thread_current ()->pages, &fake_page_entry.elem);
+
+  if (e == NULL)
+      return NULL;
+
+  return hash_entry (e, struct page_entry, elem);
+}
+
+bool
+page_load (struct page_entry *page, bool writable)
+{
+  /* Allocate a frame for the page. If a frame allocation fails, then
+     frame_alloc should try to evict a page. If it is still NULL, the OS
+     panics as this should not happen if eviction is working correctly. */
+  struct thread *t = thread_current ();
+  void *frame = frame_alloc (0, page->upage, t);
+  pagedir_set_accessed (t->pagedir, page->upage, true);
+  if (frame == NULL)
+    PANIC ("Could not allocate a frame to load page into memory.");
+
+  /* Map the page to the frame. */
+  if (!install_page (page->upage, frame, writable))
+    {
+      frame_free (frame);
+      return false;
+    }
+
+  /* Move the file pointer to the correct location in the file. Then, read the
+     data from the file into the frame. Checks that we were able to read the
+     expected number of bytes. */
+  file_seek (page->file, page->offset);
+  if (file_read (page->file, frame, page->read_bytes) != (int) page->read_bytes)
+  {
+    frame_free (frame);
+    return false;
+  }
+
+  /* Zero out the remaining bytes in the frame. */
+  memset (frame + page->read_bytes, 0, page->zero_bytes);
+
+  /* Mark the page as loaded successfully. */
+  return true;
+}
+
+/* Function to clean up a page_entry. Given the elem of that page_entry, frees
+   the page_entry itself. */
+void
+page_cleanup (struct hash_elem *e, void *aux UNUSED)
+{
+  free (hash_entry (e, struct page_entry, elem));
+}
+
 /* Updates the 'owner' thread's page table entry for virtual address 'upage'
-   to have a present bit of 0 and stores the specified swap slot value in the
-   entry for later retrieval from disk. */
+   to flag the page as being stored in swap, and stores the specified swap slot
+   value in the entry at the address bits for later retrieval from disk. */
 void
 page_set_swap (struct thread *owner, void *upage, size_t swap_slot)
+{
+  uint32_t *pte = lookup_page (owner->pagedir, upage, false);
+
+  /* Store the provided swap slot in the address bits of the page table
+     entry, truncating excess bits. */
+  *pte |= (1 << SWAP_FLAG_BIT);
+  uint32_t swap_slot_bits = (swap_slot << ADDR_START_BIT) & PTE_ADDR;
+  *pte = (*pte & PTE_FLAGS) | swap_slot_bits;
+
+  invalidate_pagedir (owner->pagedir);
+}
+
+/* Returns true iff the page with user address 'upage' owned by 'owner' 
+   is flagged to be in the swap disk via the owner's page table. */
+bool
+page_in_swap (struct thread *owner, void *upage)
 {
    uint32_t *pte = lookup_page (owner->pagedir, upage, false);
-
-   /* Store the provided swap slot in the address bits of the page table
-      entry, truncating excess bits. */
-   *pte |= (1 << SWAP_FLAG_BIT);
-   uint32_t swap_slot_bits = (swap_slot << ADDR_START_BIT) & PTE_ADDR;
-   *pte = (*pte & PTE_FLAGS) | swap_slot_bits;
-
-   /* Mark page as 'not present' and flag the page directory as having
-      been modified. */
-   pagedir_clear_page (owner->pagedir, upage);
+   return pte != NULL &&
+          (*pte & (1 << SWAP_FLAG_BIT)) != 0;
 }
 
 /* Given that the page with user address 'upage' owned by 'owner' is flagged
@@ -32,10 +150,9 @@ page_get_swap (struct thread *owner, void *upage)
 {
    uint32_t *pte = lookup_page (owner->pagedir, upage, false);
 
+   ASSERT (pte != NULL);
    ASSERT ((*pte & PTE_P) == 0);
-   ASSERT ((*pte & (1 << SWAP_FLAG_BIT)) != 0);
 
    /* Masks the address bits and returns truncated value. */
    return ((*pte & PTE_ADDR) >> ADDR_START_BIT);
 }
-

src/vm/page.h

@@ -2,8 +2,38 @@
 #define VM_PAGE_H
 
 #include "threads/thread.h"
+#include "filesys/off_t.h"
 
+enum page_type {
+  PAGE_EXECUTABLE,
+  PAGE_EMPTY
+};
+
+struct page_entry {
+  enum page_type type; /* Type of Data that should go into the page */
+  void *upage;         /* Start Address of the User Page (Key of hash table). */
+
+  /* File Data */
+  struct file *file;   /* Pointer to the file for executables. */
+  off_t offset;        /* Offset of the page content within the file. */
+  uint32_t read_bytes; /* Number of bytes to read within the page. */
+  uint32_t zero_bytes; /* Number of bytes to zero within the page. */
+  bool writable;       /* Flag for whether this page is writable or not. */
+
+  struct hash_elem elem; /* An elem for the hash table. */
+};
+
+unsigned page_hash (const struct hash_elem *e, void *aux);
+bool page_less (const struct hash_elem *a_, const struct hash_elem *b_,
+                void *aux);
+struct page_entry *page_insert (struct file *file, off_t ofs, void *upage,
+                                uint32_t read_bytes, uint32_t zero_bytes,
+                                bool writable, enum page_type type);
+struct page_entry *page_get (void *upage);
+bool page_load (struct page_entry *page, bool writable);
+void page_cleanup (struct hash_elem *e, void *aux);
 void page_set_swap (struct thread *, void *, size_t);
+bool page_in_swap (struct thread *, void *);
 size_t page_get_swap (struct thread *, void *);
 
 #endif /* vm/frame.h */