feat: implement frame table without thread safety

fix: code review 2 changes

See merge request lab2425_autumn/pintos_22!52

.gitlab-ci.yml

@@ -16,18 +16,13 @@ stages:
   script:
     - cd src/$DIR
     - make check | tee build.log
-    - grep -q "FAIL tests/$DIR" build.log && exit 1 || exit 0
+    - grep -vE "^FAIL $IGNORE\$" build.log | grep -q "FAIL tests/$DIR" && exit 1 || exit 0
 
 test_devices:
   extends: .pintos_tests
   variables:
     DIR: devices
 
-test_filesys:
-  extends: .pintos_tests
-  variables:
-    DIR: filesys
-
 test_threads:
   extends: .pintos_tests
   variables:
@@ -42,3 +37,4 @@ test_vm:
   extends: .pintos_tests
   variables:
     DIR: vm
+    IGNORE: (tests/vm/pt-grow-stack|tests/vm/pt-grow-pusha|tests/vm/pt-big-stk-obj|tests/vm/pt-overflowstk|tests/vm/pt-write-code2|tests/vm/pt-grow-stk-sc|tests/vm/page-linear|tests/vm/page-parallel|tests/vm/page-merge-seq|tests/vm/page-merge-par|tests/vm/page-merge-stk|tests/vm/page-merge-mm|tests/vm/mmap-read|tests/vm/mmap-close|tests/vm/mmap-overlap|tests/vm/mmap-twice|tests/vm/mmap-write|tests/vm/mmap-exit|tests/vm/mmap-shuffle|tests/vm/mmap-clean|tests/vm/mmap-inherit|tests/vm/mmap-misalign|tests/vm/mmap-null|tests/vm/mmap-over-code|tests/vm/mmap-over-data|tests/vm/mmap-over-stk|tests/vm/mmap-remove)

src/Makefile.build

@@ -62,6 +62,7 @@ userprog_SRC += userprog/gdt.c		# GDT initialization.
 userprog_SRC += userprog/tss.c		# TSS management.
 
 # Virtual memory code.
+vm_SRC += vm/frame.c                    # Frame table manager.
 vm_SRC += devices/swap.c		# Swap block manager.
 #vm_SRC = vm/file.c			# Some other file.
 

src/threads/init.c

@@ -32,6 +32,7 @@
 #include "tests/threads/tests.h"
 #endif
 #ifdef VM
+#include "vm/frame.h"
 #include "devices/swap.h"
 #endif
 #ifdef FILESYS
@@ -101,6 +102,9 @@ main (void)
   palloc_init (user_page_limit);
   malloc_init ();
   paging_init ();
+#ifdef VM
+  frame_init ();
+#endif
 
   /* Segmentation. */
 #ifdef USERPROG

src/threads/thread.c

@@ -71,7 +71,7 @@ static void kernel_thread (thread_func *, void *aux);
 static void idle (void *aux UNUSED);
 static struct thread *running_thread (void);
 static struct thread *next_thread_to_run (void);
-static void init_process_result (struct thread *t);
+static bool init_process_result (struct thread *t);
 static void init_thread (struct thread *, const char *name, int nice,
                          int priority, fp32_t recent_cpu);
 static bool is_thread (struct thread *) UNUSED;
@@ -84,6 +84,10 @@ void thread_schedule_tail (struct thread *prev);
 static tid_t allocate_tid (void);
 static bool donor_priority_less (const struct list_elem *a_,
                                  const struct list_elem *b_, void *aux UNUSED);
+static unsigned process_result_hash (const struct hash_elem *e,
+                                     void *aux UNUSED);
+static bool process_result_less (const struct hash_elem *a,
+                                 const struct hash_elem *b, void *aux UNUSED);
 
 /* Initializes the threading system by transforming the code
    that's currently running into a thread.  This can't work in
@@ -122,6 +126,13 @@ thread_init (void)
 void
 thread_start (void) 
 {
+  /* Malloc has been initalised, we can allocate the child results table
+     for the main thread. */
+  struct thread *t = thread_current ();
+  if (!hash_init (&t->child_results, process_result_hash, process_result_less,
+                  t))
+    PANIC ("Failed to initialise child results table for main thread.");
+
   /* Create the idle thread. */
   struct semaphore idle_started;
   sema_init (&idle_started, 0);
@@ -241,11 +252,25 @@ thread_create (const char *name, int priority,
   struct thread *parent_thread = thread_current ();
   init_thread (t, name, parent_thread->nice, priority, parent_thread->recent_cpu);
   tid = t->tid = allocate_tid ();
-  init_process_result (t);
+  if (!init_process_result (t))
+    {
+      palloc_free_page (t);
+      return TID_ERROR;
+    }
 
-  #ifdef USERPROG
-    hash_init (&t->open_files, fd_hash, fd_less, NULL);
-  #endif
+#ifdef USERPROG
+  /* Initialize the thread's file descriptor table. */
+  t->fd_counter = MINIMUM_USER_FD;
+
+  if (!hash_init (&t->open_files, fd_hash, fd_less, NULL)
+      || !hash_init (&t->child_results, process_result_hash,
+                     process_result_less, t))
+    {
+      palloc_free_page (t);
+      free (t->result);
+      return TID_ERROR;
+    }
+#endif
 
   /* Prepare thread for first run by initializing its stack.
      Do this atomically so intermediate values for the 'stack' 
@@ -269,9 +294,7 @@ thread_create (const char *name, int priority,
 
   intr_set_level (old_level);
 
-  /* No need to synchronise child_results since it is only ever accessed by one
-     thread. By the nature of increasing TIDs, this list is ordered. */
-  list_push_back (&parent_thread->child_results, &t->result->elem);
+  hash_insert (&parent_thread->child_results, &t->result->elem);
 
   /* Add to run queue. */
   thread_unblock (t);
@@ -649,15 +672,18 @@ is_thread (struct thread *t)
 }
 
 /* Allocate and initialise a process result for given thread. */
-static void
+static bool
 init_process_result (struct thread *t)
 {
   struct process_result *result = malloc (sizeof (struct process_result));
+  if (result == NULL)
+    return false;
   result->tid = t->tid;
-  result->exit_status = t->exit_status;
+  result->exit_status = -1;
   lock_init (&result->lock);
   sema_init (&result->sema, 0);
   t->result = result;
+  return true;
 }
 
 /* Does basic initialization of T as a blocked thread named
@@ -688,10 +714,6 @@ init_thread (struct thread *t, const char *name, int nice, int priority,
   t->recent_cpu = recent_cpu;
   t->priority = t->base_priority;
 
-  t->fd_counter = MINIMUM_USER_FD;
-  t->exit_status = -1;
-  list_init (&t->child_results);
-
   old_level = intr_disable ();
   list_push_back (&all_list, &t->allelem);
   intr_set_level (old_level);
@@ -822,6 +844,29 @@ allocate_tid (void)
   return tid;
 }
 
+/* Hashing function needed for child_results table.
+   Returns hash of process_result's TID. */
+static unsigned
+process_result_hash (const struct hash_elem *e, void *aux UNUSED)
+{
+  const struct process_result *result
+      = hash_entry (e, struct process_result, elem);
+  return hash_int (result->tid);
+}
+
+/* Comparator function needed for child_results table.
+   Returns less than comparison on process_results' TIDs. */
+static bool
+process_result_less (const struct hash_elem *a_, const struct hash_elem *b_,
+                     void *aux UNUSED)
+{
+  const struct process_result *a
+      = hash_entry (a_, struct process_result, elem);
+  const struct process_result *b
+      = hash_entry (b_, struct process_result, elem);
+  return a->tid < b->tid;
+}
+
 /* Offset of `stack' member within `struct thread'.
    Used by switch.S, which can't figure it out on its own. */
 uint32_t thread_stack_ofs = offsetof (struct thread, stack);

src/threads/thread.h

@@ -44,7 +44,7 @@ struct process_result
   struct lock lock;      /* Lock the exit_status and sema. */
   struct semaphore sema; /* Semaphore to signal the parent that the exit_status
                             has been set. */
-  struct list_elem elem; /* List element for the parent's children list. */
+  struct hash_elem elem; /* Hash element for the parent's children map. */
 };
 
 /* A kernel thread or user process.
@@ -128,15 +128,13 @@ struct thread
 
     /* Process wait properties. */
     struct process_result *result;      /* Result of the process. */
-    struct list child_results;          /* List of children's of this thread
-                                           process results. */
+    struct hash child_results;          /* Map of children's of this thread
+                                           TID to process result. */
     struct file *exec_file;             /* Thread's currently running file */
 
     /* Shared between thread.c and synch.c. */
     struct list_elem elem;              /* List element. */
 
-    int exit_status;                    /* Exit Status: 0 = successful exit. */
-
 #ifdef USERPROG
     /* Owned by userprog/process.c. */
     uint32_t *pagedir;                  /* Page directory. */

src/userprog/process.c

@@ -1,5 +1,6 @@
 #include "userprog/process.h"
 #include <debug.h>
+#include <hash.h>
 #include <inttypes.h>
 #include <list.h>
 #include <round.h>
@@ -23,6 +24,9 @@
 #include "threads/vaddr.h"
 #include "threads/synch.h"
 #include "devices/timer.h"
+#ifdef VM
+#include "vm/frame.h"
+#endif
 
 /* Defines the native number of bytes processed by the processor
    (for the purposes of alignment). */
@@ -55,6 +59,7 @@ struct process_start_data
   };
 
 static thread_func start_process NO_RETURN;
+static void destruct_process_result (struct hash_elem *e, void *aux UNUSED);
 static bool load (const char *cmdline, void (**eip) (void), void **esp);
 
 /* Starts a new thread running a user program executed via
@@ -111,7 +116,10 @@ process_execute (const char *cmd)
   return tid;
 }
 
+static void *get_usr_kpage (enum palloc_flags flags);
+static void free_usr_kpage (void *kpage);
 static bool install_page (void *upage, void *kpage, bool writable);
+
 static bool process_init_stack (char *cmd_saveptr, void **esp, char *file_name);
 static void *push_to_stack (void **esp, void *data, size_t data_size);
 #define push_var_to_stack(esp, var) (push_to_stack (esp, &var, sizeof (var)))
@@ -251,7 +259,7 @@ process_init_stack (char *cmd_saveptr, void **esp, char *file_name)
         /* Allocate the pages and map them to the user process. */
         for (int i = 1; i < pages_needed + 1; i++)
           {
-            uint8_t *kpage = palloc_get_page (PAL_USER | PAL_ZERO);
+            uint8_t *kpage = get_usr_kpage (PAL_ZERO);
             if (!install_page (((uint8_t *) PHYS_BASE) - PGSIZE * (i + 1),
                                kpage, true))
               return false;
@@ -315,32 +323,15 @@ push_to_stack (void **esp, void *data, size_t data_size)
 int
 process_wait (tid_t child_tid)
 {
-  struct process_result *child_result = NULL;
-  struct list_elem *e;
-  struct thread *cur = thread_current ();
-
-  for (e = list_begin (&cur->child_results);
-       e != list_end (&cur->child_results); e = list_next (e))
-    {
-      struct process_result *result
-          = list_entry (e, struct process_result, elem);
-      if (result->tid == child_tid)
-        {
-          /* Found the child process. */
-          child_result = result;
-          break;
-        }
-
-      /* List is ordered, allowing us to break early if the child_tid is
-         greater than the current result's tid. */
-      else if (result->tid > child_tid)
-        break;
-    }
-
-  /* If the child process was not found, return -1. */
-  if (child_result == NULL)
+  struct thread *t = thread_current ();
+  struct process_result fake_result;
+  fake_result.tid = child_tid;
+  struct hash_elem *e = hash_find (&t->child_results, &fake_result.elem);
+  if (e == NULL)
     return -1;
 
+  struct process_result *child_result
+      = hash_entry (e, struct process_result, elem);
   /* Wait for child to die. */
   sema_down (&child_result->sema);
 
@@ -348,18 +339,17 @@ process_wait (tid_t child_tid)
      wait) for it here to ensure we don't free the lock memory before it is
      released in process_exit. */
   lock_acquire (&child_result->lock);
-
-  /* To prevent waiting for child twice, remove it from the list.
+  /* To prevent waiting for child twice, remove it from the table.
      No need to use lock since this is the only thread with access to
      the struct process_result now. */
-  list_remove (&child_result->elem);
+  hash_delete (&t->child_results, &child_result->elem);
 
   /* Get the exit status of the child */
   int exit_status = child_result->exit_status;
 
   /* Release the lock */
   lock_release (&child_result->lock);
-
+  /* Result no-longer used by parent, nor child. Deallocate it. */
   free (child_result);
   return exit_status;
 }
@@ -371,8 +361,6 @@ process_exit (void)
   struct thread *cur = thread_current ();
   uint32_t *pd;
 
-  printf ("%s: exit(%d)\n", cur->name, cur->exit_status);
-
   /* Clean up all open files */
   hash_destroy (&cur->open_files, fd_cleanup);
 
@@ -385,49 +373,15 @@ process_exit (void)
       lock_release (&filesys_lock);
     }
 
-  /* Update process result. */
   if (cur->result != NULL)
     {
-      lock_acquire (&cur->result->lock);
-      cur->result->exit_status = cur->exit_status;
-      /* Parent has died, child has to free the struct process_result * */
-      if (sema_try_down (&cur->result->sema))
-        {
-          lock_release (&cur->result->lock);
-          free (cur->result);
-        }
-      /* Parent is still alive and will be the one to free the
-         struct process_result *, and may be waiting so call sema_up */
-      else
-        {
-          sema_up (&cur->result->sema);
-          lock_release (&cur->result->lock);
-        }
+      printf ("%s: exit(%d)\n", cur->name, cur->result->exit_status);
+      /* Update own process result. */
+      destruct_process_result (&cur->result->elem, cur);
     }
 
   /* Free child process results or signal parent's death. */
-  struct list_elem *e;
-  for (e = list_begin (&cur->child_results);
-       e != list_end (&cur->child_results);)
-    {
-      struct process_result *result
-          = list_entry (e, struct process_result, elem);
-      struct list_elem *next = list_next (e);
-      lock_acquire (&result->lock);
-      /* Child has died (and was not waited for). Free the result. */
-      if (sema_try_down (&result->sema))
-        {
-          lock_release (&result->lock);
-          free (result);
-        }
-      /* Child is still alive, signal via sema that parent has died. */
-      else
-        {
-          sema_up (&result->sema);
-          lock_release (&result->lock);
-        }
-      e = next;
-    }
+  hash_destroy (&cur->child_results, destruct_process_result);
 
   /* Destroy the current process's page directory and switch back
      to the kernel-only page directory. */
@@ -447,6 +401,28 @@ process_exit (void)
     }
 }
 
+/* Destruct a process_result, with multi-thread awareness.
+   If the other thread is running, simply signals death. Otherwise
+   frees the result. */
+static void
+destruct_process_result (struct hash_elem *e, void *aux UNUSED)
+{
+  struct process_result *result = hash_entry (e, struct process_result, elem);
+  lock_acquire (&result->lock);
+  /* Other thread has died (and was not waited for). Free the result. */
+  if (sema_try_down (&result->sema))
+    {
+      lock_release (&result->lock);
+      free (result);
+    }
+  /* Other thread is still alive, signal via sema that parent has died. */
+  else
+    {
+      sema_up (&result->sema);
+      lock_release (&result->lock);
+    }
+}
+
 /* Sets up the CPU for running user code in the current
    thread.
    This function is called on every context switch. */
@@ -734,7 +710,7 @@ load_segment (struct file *file, off_t ofs, uint8_t *upage,
       if (kpage == NULL){
         
         /* Get a new page of memory. */
-        kpage = palloc_get_page (PAL_USER);
+        kpage = get_usr_kpage (0);
         if (kpage == NULL){
           return false;
         }
@@ -742,7 +718,7 @@ load_segment (struct file *file, off_t ofs, uint8_t *upage,
         /* Add the page to the process's address space. */
         if (!install_page (upage, kpage, writable)) 
         {
-          palloc_free_page (kpage);
+          free_usr_kpage (kpage);
           return false; 
         }     
         
@@ -777,18 +753,44 @@ setup_stack (void **esp)
   uint8_t *kpage;
   bool success = false;
 
-  kpage = palloc_get_page (PAL_USER | PAL_ZERO);
+  kpage = get_usr_kpage (PAL_ZERO);
   if (kpage != NULL) 
     {
       success = install_page (((uint8_t *) PHYS_BASE) - PGSIZE, kpage, true);
       if (success)
         *esp = PHYS_BASE;
       else
-        palloc_free_page (kpage);
+        free_usr_kpage (kpage);
     }
   return success;
 }
 
+/* Claims a page from the user pool and returns its kernel address,
+   updating the frame table if VM is enabled. */
+static void *
+get_usr_kpage (enum palloc_flags flags)
+{
+  void *page;
+#ifdef VM
+  page = frame_alloc (flags);
+#else
+  page = palloc_get_page (flags | PAL_USER);
+#endif
+  return page;
+}
+
+/* Frees a page belonging to a user process given its kernel address,
+   updating the frame table if VM is enabled. */
+static void
+free_usr_kpage (void *kpage)
+{
+#ifdef VM
+  frame_free (kpage);
+#else
+  palloc_free_page (kpage);
+#endif
+}
+
 /* Adds a mapping from user virtual address UPAGE to kernel
    virtual address KPAGE to the page table.
    If WRITABLE is true, the user process may modify the page;

src/userprog/syscall.c

@@ -131,7 +131,7 @@ syscall_exit (int status)
 {
   /* Sets exit_status of the thread to status. thread_exit () will call
      process_exit () if user programs are allowed. */
-  thread_current ()->exit_status = status;
+  thread_current ()->result->exit_status = status;
   thread_exit ();
 }
 
@@ -271,7 +271,7 @@ syscall_read (int fd, void *buffer, unsigned size)
     {
       /* Reading from the console. */
       char *write_buffer = buffer;
-      for (int i = 0; i < size; i++)
+      for (unsigned i = 0; i < size; i++)
         write_buffer[i] = input_getc ();
 
       /* In case of console, read is always (eventually) successful. So return
@@ -348,12 +348,7 @@ syscall_seek (int fd, unsigned position)
   /* Find the file from the FD. If it does not exist, do nothing. */
   struct open_file *file_info = fd_get_file (fd);
   if (file_info != NULL)
-  {
-    /* File exists: Acquire the file system lock to prevent race conditions. */
-    lock_acquire (&filesys_lock);
     file_seek (file_info->file, position);
-    lock_release (&filesys_lock);
-  }
 }
 
 /* Handles the syscall for returning the next byte in a file referenced by
@@ -367,10 +362,7 @@ syscall_tell (int fd)
   if (file_info == NULL)
     return 0;
 
-  /* Acquire the file system lock to prevent race conditions. */
-  lock_acquire (&filesys_lock);
   unsigned pos = file_tell (file_info->file);
-  lock_release (&filesys_lock);
 
   /* Return the current position in the file. */
   return pos;

src/vm/frame.c

@@ -0,0 +1,137 @@
+#include <debug.h>
+#include <hash.h>
+#include <list.h>
+
+#include "frame.h"
+#include "threads/malloc.h"
+#include "threads/synch.h"
+
+/* Hash table that maps every active frame's kernel virtual address
+   to its corresponding 'frame_metadata'.*/
+struct hash frame_table;
+
+/* Linked list of frame_metadata whose pages are predicted to currently
+   be in the working set of a process. They are not considered for
+   eviction, but are considered for demotion to the 'inactive' list. */
+struct list active_list;
+
+/* Linked list of frame_metadata whose pages are predicted to leave the
+   working set of their processes soon, so are considered for eviction.
+   Pages are considered for eviction from the tail end, and are initially
+   demoted to 'inactive' at the head. */
+struct list inactive_list;
+
+/* Synchronisation variables. */
+/* Ensures mutual exclusion to accessing the 'head' and first element of
+   'inactive_list', which is accessed every time a frame is allocated. */
+struct lock inactive_head_lock;
+
+struct frame_metadata
+  {
+     void *frame;  /* The kernel virtual address holding the frame. */
+     struct hash_elem hash_elem;  /* Tracks the position of the frame metadata
+                                     within 'frame_table', whose key is the 
+                                     kernel virtual address of the frame. */
+     struct list_elem list_elem;  /* Tracks the position of the frame metadata
+                                     in either the 'active' or 'inactive' list,
+                                     so a victim can be chosen for eviction. */
+  };
+
+hash_hash_func frame_metadata_hash;
+hash_less_func frame_metadata_less;
+
+/* Initialize the frame system by initializing the frame (hash) table with
+   the frame_metadata hashing and comparison functions, as well as initializing
+   the active & inactive lists. Also initializes the system's synchronisation
+   primitives. */
+void
+frame_init (void)
+{
+  hash_init (&frame_table, frame_metadata_hash, frame_metadata_less, NULL);
+  list_init (&active_list);
+  list_init (&inactive_list);
+
+  lock_init (&inactive_head_lock);
+}
+
+/* Attempt to allocate a frame for a user process, either by direct
+   allocation of a user page if there is sufficient RAM, or by
+   evicting a currently active page if memory allocated for user
+   processes is fulled and storing it in swap. If swap is full in
+   the former case, panic the kernel. */
+void *
+frame_alloc (enum palloc_flags flags)
+{
+  flags |= PAL_USER;
+
+  void *frame = palloc_get_page (flags);
+  if (frame == NULL)
+    {
+      /* TODO: Find victim page to replace, and swap it with this new page. */
+      return NULL;
+    }
+
+  struct frame_metadata *frame_metadata =
+    malloc (sizeof (struct frame_metadata));
+  frame_metadata->frame = frame;
+
+  /* Newly faulted pages begin at the head of the inactive list. */
+  lock_acquire (&inactive_head_lock);
+  list_push_front (&inactive_list, &frame_metadata->list_elem);
+  lock_release (&inactive_head_lock);
+
+  /* Finally, insert frame metadata within the frame table, with the key as its
+     allocated kernel address. */
+  hash_replace (&frame_table, &frame_metadata->hash_elem);
+
+  return frame;
+}
+
+/* Attempt to deallocate a frame for a user process by removing it from the
+   frame table as well as active/inactive list, and freeing the underlying
+   page memory. Panics if the frame isn't active in memory. */
+void
+frame_free (void *frame)
+  {
+    struct frame_metadata key_metadata;
+    key_metadata.frame = frame;
+
+    struct hash_elem *e =
+      hash_delete (&frame_table, &key_metadata.hash_elem);
+    if (e == NULL) PANIC ("Attempted to free a frame without a corresponding "
+                          "kernel address!\n");
+
+    struct frame_metadata *frame_metadata =
+      hash_entry (e, struct frame_metadata, hash_elem);
+    
+    list_remove (&frame_metadata->list_elem);
+    free (frame_metadata);
+    palloc_free_page (frame);
+  }
+
+/* Hash function for frame metadata, used for storing entries in the
+   frame table. */
+unsigned
+frame_metadata_hash (const struct hash_elem *e, void *aux UNUSED)
+  {
+    struct frame_metadata *frame_metadata = 
+      hash_entry (e, struct frame_metadata, hash_elem);
+
+    return hash_bytes (&frame_metadata->frame, sizeof (frame_metadata->frame));
+  }
+
+/* 'less_func' comparison function for frame metadata, used for comparing 
+   the keys of the frame table. Returns true iff the kernel virtual address
+   of the first frame is less than that of the second frame. */
+bool
+frame_metadata_less (const struct hash_elem *a_, const struct hash_elem *b_,
+                     void *aux UNUSED)
+  {
+    struct frame_metadata *a =
+      hash_entry (a_, struct frame_metadata, hash_elem);
+    struct frame_metadata *b =
+      hash_entry (b_, struct frame_metadata, hash_elem);
+
+    return a->frame < b->frame;
+  }
+

src/vm/frame.h

@@ -0,0 +1,10 @@
+#ifndef VM_FRAME_H
+#define VM_FRAME_H
+
+#include "threads/palloc.h"
+
+void frame_init (void);
+void *frame_alloc (enum palloc_flags);
+void frame_free (void *frame);
+
+#endif /* vm/frame.h */