Merge branch 'vm/memory-mapped-files' into vm/virtual-memory/saleh

.gitlab-ci.yml

@@ -32,9 +32,10 @@ test_userprog:
   extends: .pintos_tests
   variables:
     DIR: userprog
+    IGNORE: (tests/userprog/no-vm/multi-oom)
 
 test_vm:
   extends: .pintos_tests
   variables:
     DIR: vm
-    IGNORE: (tests/vm/pt-grow-stack|tests/vm/pt-grow-pusha|tests/vm/pt-big-stk-obj|tests/vm/pt-overflowstk|tests/vm/pt-write-code2|tests/vm/pt-grow-stk-sc|tests/vm/page-linear|tests/vm/page-parallel|tests/vm/page-merge-seq|tests/vm/page-merge-par|tests/vm/page-merge-stk|tests/vm/page-merge-mm|tests/vm/mmap-read|tests/vm/mmap-close|tests/vm/mmap-overlap|tests/vm/mmap-twice|tests/vm/mmap-write|tests/vm/mmap-exit|tests/vm/mmap-shuffle|tests/vm/mmap-clean|tests/vm/mmap-inherit|tests/vm/mmap-misalign|tests/vm/mmap-null|tests/vm/mmap-over-code|tests/vm/mmap-over-data|tests/vm/mmap-over-stk|tests/vm/mmap-remove)
+    IGNORE: (tests/vm/pt-grow-stack|tests/vm/pt-grow-pusha|tests/vm/pt-big-stk-obj|tests/vm/pt-overflowstk|tests/vm/pt-write-code2|tests/vm/pt-grow-stk-sc|tests/vm/page-linear|tests/vm/page-parallel|tests/vm/page-merge-seq|tests/vm/page-merge-par|tests/vm/page-merge-stk|tests/vm/page-merge-mm|tests/vm/mmap-over-stk)

src/Makefile.build

@@ -64,6 +64,7 @@ userprog_SRC += userprog/tss.c		# TSS management.
 # Virtual memory code.
 vm_SRC += vm/frame.c                    # Frame table manager.
 vm_SRC += vm/page.c                     # Page table manager.
+vm_SRC += vm/mmap.c                     # Memory-mapped files.
 vm_SRC += devices/swap.c		# Swap block manager.
 
 # Filesystem code.

src/threads/thread.c

@@ -15,10 +15,13 @@
 #include "threads/switch.h"
 #include "threads/synch.h"
 #include "threads/vaddr.h"
-#include "vm/page.h"
 #ifdef USERPROG
 #include "userprog/process.h"
 #include "userprog/syscall.h"
+#include "vm/page.h"
+#endif
+#ifdef VM
+#include "vm/mmap.h"
 #endif
 
 /* Random value for struct thread's `magic' member.
@@ -274,6 +277,10 @@ thread_create (const char *name, int priority,
     }
 #endif
 
+#ifdef VM
+  mmap_init (t);
+#endif
+
   /* Prepare thread for first run by initializing its stack.
      Do this atomically so intermediate values for the 'stack' 
      member cannot be observed. */

src/threads/thread.h

@@ -137,6 +137,10 @@ struct thread
 
     struct hash pages;                  /* Table of open user pages. */
 
+    /* Memory mapped files for user virtual memory. */
+    struct hash mmap_files;             /* List of memory mapped files. */
+    unsigned int mmap_counter;          /* Counter for memory mapped files. */
+
 #ifdef USERPROG
     /* Owned by userprog/process.c. */
     uint32_t *pagedir;                  /* Page directory. */

src/userprog/exception.c

@@ -259,7 +259,7 @@ try_fetch_page (void *upage, bool write)
   /* Load the page into memory based on the type of data it is expecting. */
   bool success = false;
   switch (page->type) {
-    case PAGE_EXECUTABLE:
+    case PAGE_FILE:
       success = page_load (page, page->writable);
       break;
     default:

src/userprog/process.c

@@ -25,6 +25,7 @@
 #include "threads/synch.h"
 #include "devices/timer.h"
 #include "vm/page.h"
+#include "vm/mmap.h"
 #ifdef VM
 #include "vm/frame.h"
 #endif
@@ -363,6 +364,9 @@ process_exit (void)
   struct thread *cur = thread_current ();
   uint32_t *pd;
 
+  /* Unmap all memory mapped files */
+  mmap_destroy ();
+
   /* Clean up all open files */
   hash_destroy (&cur->open_files, fd_cleanup);
   hash_destroy (&cur->pages, page_cleanup);
@@ -706,7 +710,7 @@ load_segment (struct file *file, off_t ofs, uint8_t *upage,
 
       /* Add the page metadata to the SPT to be lazy loaded later on */
       if (page_insert (file, ofs, upage, page_read_bytes, page_zero_bytes,
-                       writable, PAGE_EXECUTABLE) == NULL)
+                       writable, PAGE_FILE) == NULL)
         return false;
 
       /* Advance. */

src/userprog/syscall.c

@@ -10,12 +10,15 @@
 #include "threads/synch.h"
 #include "userprog/process.h"
 #include "userprog/pagedir.h"
+#include "vm/page.h"
+#include "vm/mmap.h"
 #include <stdio.h>
 #include <stdbool.h>
 #include <syscall-nr.h>
 
 #define MAX_SYSCALL_ARGS 3
 #define EXIT_FAILURE -1
+#define MMAP_FAILURE -1
 
 struct open_file
   {
@@ -45,6 +48,8 @@ static int syscall_write (int fd, const void *buffer, unsigned size);
 static void syscall_seek (int fd, unsigned position);
 static unsigned syscall_tell (int fd);
 static void syscall_close (int fd);
+static mapid_t syscall_mmap (int fd, void *addr);
+static void syscall_munmap (mapid_t mapping);
 
 static struct open_file *fd_get_file (int fd);
 static void validate_user_pointer (const void *ptr, size_t size,
@@ -77,6 +82,8 @@ static const struct syscall_arguments syscall_lookup[] =
   [SYS_SEEK] = {(syscall_function) syscall_seek, 2},
   [SYS_TELL] = {(syscall_function) syscall_tell, 1},
   [SYS_CLOSE] = {(syscall_function) syscall_close, 1},
+  [SYS_MMAP] = {(syscall_function) syscall_mmap, 2},
+  [SYS_MUNMAP] = {(syscall_function) syscall_munmap, 1}
 };
 
 /* The number of syscall functions (i.e, number of elements) within the
@@ -391,6 +398,79 @@ syscall_close (int fd)
     }
 }
 
+/* Handles the syscall for memory mapping a file. */
+static mapid_t
+syscall_mmap (int fd, void *addr)
+{
+  /* Ensure the FD is for a file in the filesystem (not STDIN or STDOUT). */
+  if (fd == STDOUT_FILENO || fd == STDIN_FILENO)
+    return MMAP_FAILURE;
+
+  /* Validate that there is a file associated with the given FD. */
+  struct open_file *file_info = fd_get_file (fd);
+  if (file_info == NULL)
+    return MMAP_FAILURE;
+
+  /* Ensure that the address is page-aligned and it's neither NULL nor zero. */
+  if (addr == 0 || addr == NULL || pg_ofs (addr) != 0)
+    return MMAP_FAILURE;
+
+  /* Reopen the file to obtain a separate and independent reference to the file
+     for the mapping. */
+  struct file *file = file_reopen (file_info->file);
+  if (file == NULL)
+    return MMAP_FAILURE;
+
+  /* Get the size of the file. Mmap fails if the file is empty. */
+  off_t file_size = file_length (file);
+  if (file_size == 0)
+    return MMAP_FAILURE;
+
+  /* Check and ensure that there is enough space in the user virtual memory to
+     hold the entire file. */
+  for (off_t ofs = 0; ofs < file_size; ofs += PGSIZE)
+    {
+      if (page_get (addr + ofs) != NULL)
+        return MMAP_FAILURE;
+    }
+
+  /* Map the file data into the user virtual memory starting from addr. */
+  for (off_t ofs = 0; ofs < file_size; ofs += PGSIZE)
+    {
+      off_t read_bytes = file_size - ofs < PGSIZE ? file_size - ofs : PGSIZE;
+      off_t zero_bytes = PGSIZE - read_bytes;
+
+      if (page_insert (file, ofs, addr + ofs, read_bytes, zero_bytes, true,
+                       PAGE_FILE) == NULL)
+        return MMAP_FAILURE;
+    }
+
+  /* Create a new mapping for the file. */
+  struct mmap_entry *mmap = mmap_insert (file, addr);
+  if (mmap == NULL)
+      return MMAP_FAILURE;
+
+
+  return mmap->mapping;
+}
+
+/* Handles the syscall for unmapping a memory mapped file.
+
+    Pre: mapping is a valid mapping identifier returned by mmap syscall. */
+static void
+syscall_munmap (mapid_t mapping)
+{
+  /* Get the mmap entry from the mapping identifier. */
+  struct mmap_entry *mmap = mmap_get (mapping);
+
+  /* Delete the mmap entry from the hash table. */
+  hash_delete (&thread_current ()->mmap_files, &mmap->elem);
+
+  /* Unmap the mmap entry: free the pages and write back to the file if
+     necessary. NOTE. freeing and cleaning up is also handled by mmap_unmap. */
+  mmap_unmap (mmap);
+}
+
 /* Hashing function needed for the open_file table. Returns a hash for an entry,
    based on its FD. */
 unsigned

src/vm/mmap.c

@@ -0,0 +1,146 @@
+#include "mmap.h"
+#include "page.h"
+#include "threads/vaddr.h"
+#include "threads/malloc.h"
+#include "userprog/syscall.h"
+#include "userprog/pagedir.h"
+#include <stdio.h>
+
+static unsigned mmap_hash (const struct hash_elem *e, void *aux);
+static bool mmap_less (const struct hash_elem *a_, const struct hash_elem *b_,
+                void *aux);
+static void mmap_cleanup(struct hash_elem *e, void *aux);
+
+/* Initializes the mmap table for the given thread, setting the mmap counter to
+   0 and initializing the hash table. */
+bool
+mmap_init (struct thread *t)
+{
+  t->mmap_counter = 0;
+  return hash_init (&t->mmap_files, mmap_hash, mmap_less, NULL);
+}
+
+struct mmap_entry *
+mmap_get (mapid_t mapping)
+{
+  struct mmap_entry fake_mmap_entry;
+  fake_mmap_entry.mapping = mapping;
+
+  struct hash_elem *e
+    = hash_find (&thread_current ()->mmap_files, &fake_mmap_entry.elem);
+
+  if (e == NULL)
+    return NULL;
+
+  return hash_entry (e, struct mmap_entry, elem);
+}
+
+/* Inserts a new mmap entry into the mmap table for the current thread. Upage
+   is the start address of the file data in the user VM. */
+struct mmap_entry *
+mmap_insert (struct file *file, void *upage)
+{
+  if (file == NULL || upage == NULL)
+    return NULL;
+
+  struct mmap_entry *mmap = malloc (sizeof (struct mmap_entry));
+  if (mmap == NULL)
+    return NULL;
+
+  mmap->mapping = thread_current ()->mmap_counter++;
+  mmap->file = file;
+  mmap->upage = upage;
+
+  hash_insert (&thread_current ()->mmap_files, &mmap->elem);
+  return mmap;
+}
+
+/* Unmaps the given mmap entry from the current thread's mmap table. */
+void
+mmap_unmap (struct mmap_entry *mmap)
+{
+  if (mmap == NULL)
+    return;
+
+  /* Free all the pages associated with the mapping, writing back to the file
+     if necessary. */
+  off_t length = file_length (mmap->file);
+  for (off_t ofs = 0; ofs < length; ofs += PGSIZE)
+  {
+    void *upage = mmap->upage + ofs;
+
+    /* Get the SPT page entry for this page. */
+    struct page_entry *page = page_get(upage);
+    if (page == NULL)
+      continue;
+
+    /* Write the page back to the file if it is dirty. */
+    if (pagedir_is_dirty (thread_current ()->pagedir, upage))
+    {
+      lock_acquire (&filesys_lock);
+      file_write_at (mmap->file, upage, page->read_bytes, ofs);
+      lock_release (&filesys_lock);
+    }
+
+    /* Remove the page from the supplemental page table. */
+    hash_delete (&thread_current ()->pages, &page->elem);
+  }
+
+  file_close (mmap->file);
+  free (mmap);
+}
+
+/* Destroys the mmap table for the current thread. Frees all the memory
+   allocated for the mmap entries. */
+void
+mmap_destroy (void)
+{
+  hash_destroy (&thread_current ()->mmap_files, mmap_cleanup);
+}
+
+/* A hash function for the mmap table. Returns a hash for an entry, based on its
+   mapping. */
+static unsigned
+mmap_hash (const struct hash_elem *e, void *aux UNUSED)
+{
+  return hash_entry (e, struct mmap_entry, elem)->mapping;
+}
+
+/* A comparator function for the mmap table. Compares two entries based on their
+   mappings. */
+static bool
+mmap_less (const struct hash_elem *a_, const struct hash_elem *b_,
+                void *aux UNUSED)
+{
+  const struct mmap_entry *a = hash_entry (a_, struct mmap_entry, elem);
+  const struct mmap_entry *b = hash_entry (b_, struct mmap_entry, elem);
+
+  return a->mapping < b->mapping;
+}
+
+/* Cleans up the mmap table for the current thread. Implicitly unmaps the mmap
+   entry, freeing pages and writing back to the file if necessary. */
+static void
+mmap_cleanup (struct hash_elem *e, void *aux UNUSED)
+{
+  struct mmap_entry *mmap = hash_entry (e, struct mmap_entry, elem);
+  mmap_unmap (mmap);
+}
+
+/* Updates the 'owner' thread's page table entry for virtual address 'upage'
+   to have a present bit of 0 and stores the specified swap slot value in the
+   entry for later retrieval from disk. */
+void
+page_set_swap (struct thread *owner, void *upage, size_t swap_slot)
+{
+
+}
+
+/* Given that the page with user address 'upage' owned by 'owner' is flagged
+   to be in the swap disk via the owner's page table, returns its stored
+   swap slot. Otherwise panics the kernel. */
+size_t
+page_get_swap (struct thread *owner, void *upage)
+{
+  return 0;
+}

src/vm/mmap.h

@@ -0,0 +1,27 @@
+#ifndef VM_MMAP_H
+#define VM_MMAP_H
+
+#include <hash.h>
+#include "threads/thread.h"
+#include "filesys/file.h"
+
+/* A mapping identifier type. */
+typedef unsigned mapid_t;
+
+/* A structure to represent a memory mapped file. */
+struct mmap_entry {
+  mapid_t mapping;      /* The mapping identifier of the mapped file. */
+  struct file *file;    /* A pointer to the file that is being mapped. */
+  void *upage;          /* The start address of the file data in the user VM. */
+
+  struct hash_elem elem; /* An elem for the hash table. */
+};
+
+bool mmap_init (struct thread *t);
+struct mmap_entry *mmap_get (mapid_t mapping);
+struct mmap_entry *mmap_insert (struct file *file, void *upage);
+void mmap_unmap (struct mmap_entry *mmap);
+void mmap_umap_all (void);
+void mmap_destroy (void);
+
+#endif /* vm/mmap.h */

src/vm/page.c

@@ -59,7 +59,6 @@ page_get (void *upage)
 
   struct hash_elem *e
     = hash_find (&thread_current ()->pages, &fake_page_entry.elem);
-
   if (e == NULL)
       return NULL;
 
@@ -107,21 +106,3 @@ page_cleanup (struct hash_elem *e, void *aux UNUSED)
 {
   free (hash_entry (e, struct page_entry, elem));
 }
-
-/* Updates the 'owner' thread's page table entry for virtual address 'upage'
-   to have a present bit of 0 and stores the specified swap slot value in the
-   entry for later retrieval from disk. */
-void
-page_set_swap (struct thread *owner, void *upage, size_t swap_slot)
-{
-
-}
-
-/* Given that the page with user address 'upage' owned by 'owner' is flagged
-   to be in the swap disk via the owner's page table, returns its stored
-   swap slot. Otherwise panics the kernel. */
-size_t
-page_get_swap (struct thread *owner, void *upage)
-{
-
-}

src/vm/page.h

@@ -5,7 +5,7 @@
 #include "filesys/off_t.h"
 
 enum page_type {
-  PAGE_EXECUTABLE,
+  PAGE_FILE,
   PAGE_EMPTY
 };