Half boot time on pi when using standalone by using greffs

Makefile.util.def

@@ -119,6 +119,7 @@ library = {
   common = grub-core/fs/sfs.c;
   common = grub-core/fs/squash4.c;
   common = grub-core/fs/tar.c;
+  common = grub-core/fs/greffs.c;
   common = grub-core/fs/udf.c;
   common = grub-core/fs/ufs2.c;
   common = grub-core/fs/ufs.c;

grub-core/Makefile.core.def

@@ -1399,6 +1399,11 @@ module = {
   common = fs/tar.c;
 };
 
+module = {
+  name = greffs;
+  common = fs/greffs.c;
+};
+
 module = {
   name = udf;
   common = fs/udf.c;

grub-core/fs/greffs.c

@@ -0,0 +1,314 @@
+/*
+ *  GRUB  --  GRand Unified Bootloader
+ *  Copyright (C) 2013 Free Software Foundation, Inc.
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <grub/misc.h>
+#include <grub/disk.h>
+#include <grub/archelp.h>
+
+#include <grub/file.h>
+#include <grub/mm.h>
+#include <grub/dl.h>
+#include <grub/i18n.h>
+#include <grub/greffs.h>
+
+GRUB_MOD_LICENSE ("GPLv3+");
+
+struct grub_greffs_data
+{
+  grub_uint32_t dofs;
+};
+
+static grub_err_t
+get_string (grub_disk_t disk,
+	    const struct grub_greffs_header *head,
+	    grub_uint32_t fn,
+	    char **buf, grub_size_t *size)
+{
+  grub_uint32_t desc[2];
+  grub_size_t read_size;
+
+  if (grub_disk_read (disk, 0,
+		      grub_cpu_to_le32 (head->string_ptrs_offset)
+		      + fn * sizeof (desc[0]), sizeof (desc), &desc))
+    return grub_errno;
+  read_size = desc[1] - desc[0];
+  if (*size < read_size + 1)
+    {
+      grub_free (*buf);
+      *size = (read_size + 4) * 2;
+      *buf = grub_malloc (*size);
+      if (!*buf)
+	{
+	  *size = 0;
+	  return grub_errno;
+	}
+    }
+  if (grub_disk_read (disk, 0, desc[0], read_size, *buf))
+    return grub_errno;
+  (*buf)[read_size] = '\0';
+  return GRUB_ERR_NONE;
+}
+
+static grub_err_t
+find_file (grub_disk_t disk,
+	   struct grub_greffs_header *head,
+	   const char *name_in, grub_uint32_t *f, int exact)
+{
+  grub_uint32_t num_files, cur_file = 0;
+  int i;
+  char *buf = NULL;
+  grub_size_t buf_size = 0;
+  grub_err_t err;
+
+  num_files = grub_le_to_cpu32 (head->nfiles);
+
+  for (i = 31; i >= 0; i--)
+    {
+      int cmp;
+      if ((cur_file | (1 << i)) > num_files)
+	continue;
+      err = get_string (disk, head, (cur_file | (1 << i)) - 1, &buf, &buf_size);
+      if (err)
+	{
+	  grub_free (buf);
+	  return err;
+	}
+      cmp = grub_strcmp (buf, name_in);
+      if (cmp <= 0)
+	cur_file |= (1 << i);
+      if (cmp == 0)
+	{
+	  grub_free (buf);
+	  *f = cur_file - 1;
+	  return GRUB_ERR_NONE;
+	}
+    }
+
+  grub_free (buf);
+  if (!exact)
+    {
+      *f = cur_file;
+      return GRUB_ERR_NONE;
+    }
+  return grub_error (GRUB_ERR_FILE_NOT_FOUND, N_("file `%s' not found"),
+		     name_in);
+}
+
+static char *
+canonicalize (const char *name_in, int make_dir)
+{
+  const char *iptr;
+  char *out, *optr;
+  out = grub_malloc (grub_strlen (name_in) + 2);
+  if (!out)
+    return NULL;
+  for (iptr = name_in, optr = out; *iptr; )
+    {
+      while (*iptr == '/')
+	iptr++;
+      if (iptr[0] == '.' && (iptr[1] == '/' || iptr[1] == 0))
+	{
+	  iptr += 2;
+	  continue;
+	}
+      if (iptr[0] == '.' && iptr[1] == '.' && (iptr[2] == '/' || iptr[2] == 0))
+	{
+	  iptr += 3;
+	  if (optr == out)
+	    continue;
+	  for (optr -= 2; optr >= out && *optr != '/'; optr--);
+	  optr++;
+	  continue;
+	}
+      while (*iptr && *iptr != '/')
+	*optr++ = *iptr++;
+      if (*iptr)
+	*optr++ = *iptr++;
+      else if (make_dir && optr != out)
+	*optr++ = '/';
+    }
+  *optr = 0;
+  return out;
+}
+
+static grub_err_t
+grub_greffs_dir (grub_device_t device, const char *path_in,
+	       grub_fs_dir_hook_t hook, void *hook_data)
+{
+  grub_err_t err;
+  grub_uint32_t cur_file, num_files;
+  char *buf = 0;
+  grub_size_t buf_size = 0;
+  char *can;
+  grub_size_t len;
+  struct grub_greffs_header head;
+
+  if (grub_disk_read (device->disk, 0, 0, sizeof (head), &head))
+    return grub_error (GRUB_ERR_BAD_FS, "not a greffs filesystem");
+
+  if (grub_memcmp (head.magic, GRUB_GREFFS_MAGIC, sizeof (head.magic)) != 0)
+    return grub_error (GRUB_ERR_BAD_FS, "not a greffs filesystem");
+
+  can = canonicalize (path_in, 1);
+  if (!can)
+    return grub_errno;
+
+  if (can[0] == '\0')
+    cur_file = 0;
+  else
+    {
+      err = find_file (device->disk, &head, can, &cur_file, 0);
+      if (err)
+	goto fail;
+    }
+
+  num_files = grub_le_to_cpu32 (head.nfiles);
+
+  len = grub_strlen (can);
+
+  while (cur_file < num_files)
+    {
+      char *p, *n;
+      struct grub_dirhook_info info;
+
+      err = get_string (device->disk, &head, cur_file, &buf, &buf_size);
+      if (err)
+	goto fail;
+      if (grub_memcmp (can, buf, len) != 0)
+	break;
+      grub_memset (&info, 0, sizeof (info));
+
+      n = buf + len;
+      while (*n == '/')
+	n++;
+
+      p = grub_strchr (n, '/');
+      if (p)
+	*p = 0;
+      info.dir = (p != NULL);
+      if (hook (n, &info, hook_data))
+	goto fail;
+      if (!p)
+	cur_file++;
+      else
+	{
+	  *p = '/' + 1;
+	  p[1] = '\0';
+	  err = find_file (device->disk, &head, buf, &cur_file, 0);
+	  if (err)
+	    goto fail;
+	}	
+    }
+
+ fail:
+  grub_free (buf);
+  grub_free (can);
+  return grub_errno;
+}
+
+
+static grub_err_t
+grub_greffs_open (grub_file_t file, const char *name_in)
+{
+  struct grub_greffs_header head;
+  struct grub_greffs_data *data;
+  struct grub_greffs_inode inode;
+  grub_err_t err;
+  grub_uint32_t cur_file;
+  char *can;
+
+  if (grub_disk_read (file->device->disk, 0, 0, sizeof (head), &head))
+    return grub_error (GRUB_ERR_BAD_FS, "not a greffs filesystem");
+
+  if (grub_memcmp (head.magic, GRUB_GREFFS_MAGIC, sizeof (head.magic)) != 0)
+    return grub_error (GRUB_ERR_BAD_FS, "not a greffs filesystem");
+
+  can = canonicalize (name_in, 0);
+  if (!can)
+    return grub_errno;
+
+  err = find_file (file->device->disk, &head, can, &cur_file, 1);
+  grub_free (can);
+  if (err)
+    return err;
+
+  data = grub_malloc (sizeof (*data));
+  if (!data)
+    return grub_errno;
+  if (grub_disk_read (file->device->disk,
+		      0, grub_le_to_cpu32 (head.inodes_offset)
+		      + sizeof (inode) * cur_file, sizeof (inode), &inode))
+    return grub_error (GRUB_ERR_BAD_FS, "not a greffs filesystem");
+
+  data->dofs = grub_cpu_to_le32 (inode.start);
+  file->size = grub_cpu_to_le32 (inode.size);
+
+  file->data = data;
+
+  return GRUB_ERR_NONE;
+}
+
+static grub_ssize_t
+grub_greffs_read (grub_file_t file, char *buf, grub_size_t len)
+{
+  struct grub_greffs_data *data;
+  grub_ssize_t ret;
+
+  data = file->data;
+
+  file->device->disk->read_hook = file->read_hook;
+  file->device->disk->read_hook_data = file->read_hook_data;
+  ret = (grub_disk_read (file->device->disk, 0, data->dofs + file->offset,
+			 len, buf)) ? -1 : (grub_ssize_t) len;
+  file->device->disk->read_hook = 0;
+
+  return ret;
+}
+
+static grub_err_t
+grub_greffs_close (grub_file_t file)
+{
+  struct grub_greffs_data *data;
+
+  data = file->data;
+  grub_free (data);
+
+  return grub_errno;
+}
+
+static struct grub_fs grub_greffs_fs = {
+  .name = "greffs",
+  .dir = grub_greffs_dir,
+  .open = grub_greffs_open,
+  .read = grub_greffs_read,
+  .close = grub_greffs_close,
+#ifdef GRUB_UTIL
+  .reserved_first_sector = 0,
+  .blocklist_install = 0,
+#endif
+};
+
+GRUB_MOD_INIT (greffs)
+{
+  grub_fs_register (&grub_greffs_fs);
+}
+
+GRUB_MOD_FINI (greffs)
+{
+  grub_fs_unregister (&grub_greffs_fs);
+}

include/grub/greffs.h

@@ -0,0 +1,55 @@
+/*
+ *  GRUB  --  GRand Unified Bootloader
+ *  Copyright (C) 2013  Free Software Foundation, Inc.
+ *
+ *  GRUB is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  GRUB is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GRUB_GREFFS_HEADER
+#define GRUB_GREFFS_HEADER	1
+
+#include <grub/types.h>
+
+/*
+  Layout:
+  header
+  grub_uint32_t[] pointers to names
+  inodes
+  names
+  contents.
+  Everything is little-endian.
+ */
+
+struct grub_greffs_header
+{
+  char magic[4];
+#define GRUB_GREFFS_MAGIC "gref"
+  grub_uint32_t nfiles;
+  /* must be divisible by 4.  */
+  grub_uint32_t inodes_offset;
+  /* must be divisible by 4.  */
+  grub_uint32_t string_ptrs_offset;
+};
+
+struct grub_greffs_inode
+{
+  grub_uint32_t start;
+  grub_uint32_t size;
+  grub_uint32_t mtime;
+  /* Currently always 0. If we ever need symlinks,
+     it could be added.  */
+  grub_uint32_t type;
+};
+
+#endif

util/grub-mkstandalone.c

@@ -21,6 +21,7 @@
 #include <grub/util/install.h>
 #include <grub/util/misc.h>
 #include <grub/emu/config.h>
+#include <grub/greffs.h>
 
 #include <string.h>
 
@@ -34,7 +35,6 @@ static char *output_image;
 static char **files;
 static int nfiles;
 const struct grub_install_image_target_desc *format;
-static FILE *memdisk;
 
 enum
   {
@@ -116,91 +116,56 @@ struct argp argp = {
   NULL, help_filter, NULL
 };
 
-/* tar support */
-#define MAGIC	"ustar"
-struct head
+struct file_desc
 {
-  char name[100];
-  char mode[8];
-  char uid[8];
-  char gid[8];
-  char size[12];
-  char mtime[12];
-  char chksum[8];
-  char typeflag;
-  char linkname[100];
-  char magic[6];
-  char version[2];
-  char uname[32];
-  char gname[32];
-  char devmajor[8];
-  char devminor[8];
-  char prefix[155];
-  char pad[12];
-} GRUB_PACKED;
+  char *name;
+  char *source;
+  grub_size_t size;
+  grub_size_t mtime;
+};
+static struct file_desc *file_descs;
+static size_t n_file_descs, alloc_file_descs;
 
-static void
-write_zeros (unsigned rsz)
+static inline void
+canonicalize (char *name)
 {
-  char buf[512];
-
-  memset (buf, 0, 512);
-  fwrite (buf, 1, rsz, memdisk);
-}
-
-static void
-write_pad (unsigned sz)
-{
-  write_zeros ((~sz + 1) & 511);
-}
-
-static void
-set_tar_value (char *field, grub_uint32_t val,
-	       unsigned len)
-{
-  unsigned i;
-  for (i = 0; i < len - 1; i++)
-    field[len - 2 - i] = '0' + ((val >> (3 * i)) & 7);
-}
-
-static void
-compute_checksum (struct head *hd)
-{
-  unsigned int chk = 0;
-  unsigned char *ptr;
-  memset (hd->chksum, ' ', 8);
-  for (ptr = (unsigned char *) hd; ptr < (unsigned char *) (hd + 1); ptr++)
-    chk += *ptr;
-  set_tar_value (hd->chksum, chk, 8);
+  char *iptr, *optr;
+  for (iptr = name, optr = name; *iptr; )
+    {
+      while (*iptr == '/')
+	iptr++;
+      if (iptr[0] == '.' && (iptr[1] == '/' || iptr[1] == 0))
+	{
+	  iptr += 2;
+	  continue;
+	}
+      if (iptr[0] == '.' && iptr[1] == '.' && (iptr[2] == '/' || iptr[2] == 0))
+	{
+	  iptr += 3;
+	  if (optr == name)
+	    continue;
+	  for (optr -= 2; optr >= name && *optr != '/'; optr--);
+	  optr++;
+	  continue;
+	}
+      while (*iptr && *iptr != '/')
+	*optr++ = *iptr++;
+      if (*iptr)
+	*optr++ = *iptr++;
+    }
+  *optr = 0;
 }
 
 static void
 add_tar_file (const char *from,
 	      const char *to)
 {
-  char *tcn;
-  const char *iptr;
-  char *optr;
-  struct head hd;
   grub_util_fd_t in;
-  ssize_t r;
-  grub_uint32_t mtime = 0;
-  grub_uint32_t size;
-
-  COMPILE_TIME_ASSERT (sizeof (hd) == 512);
+  size_t idx;
 
   if (grub_util_is_special_file (from))
     return;
 
-  mtime = grub_util_get_mtime (from);
-
-  optr = tcn = xmalloc (strlen (to) + 1);
-  for (iptr = to; *iptr == '/'; iptr++);
-  for (; *iptr; iptr++)
-    if (!(iptr[0] == '/' && iptr[1] == '/'))
-      *optr++ = *iptr;
-  *optr = '\0';
-
   if (grub_util_is_directory (from))
     {
       grub_util_fd_dir_t d;
@@ -221,69 +186,138 @@ add_tar_file (const char *from,
 	  free (fp);
 	}
       grub_util_fd_closedir (d);
-      free (tcn);
       return;
     }
 
-  if (optr - tcn > 99)
+  idx = n_file_descs++;
+  if (idx >= alloc_file_descs)
     {
-      memset (&hd, 0, sizeof (hd));
-      memcpy (hd.name, tcn, 99);
-      memcpy (hd.mode, "0000600", 7);
-      memcpy (hd.uid, "0001750", 7);
-      memcpy (hd.gid, "0001750", 7);
-
-      set_tar_value (hd.size, optr - tcn, 12);
-      set_tar_value (hd.mtime, mtime, 12);
-      hd.typeflag = 'L';
-      memcpy (hd.magic, "ustar  ", 7);
-      memcpy (hd.uname, "grub", 4);
-      memcpy (hd.gname, "grub", 4);
-
-      compute_checksum (&hd);
-
-      fwrite (&hd, 1, sizeof (hd), memdisk);
-      fwrite (tcn, 1, optr - tcn, memdisk);
-
-      write_pad (optr - tcn);
+      alloc_file_descs = 2 * n_file_descs;
+      file_descs = xrealloc (file_descs, alloc_file_descs
+			     * sizeof (file_descs[0]));
     }
 
   in = grub_util_fd_open (from, GRUB_UTIL_FD_O_RDONLY);
   if (!GRUB_UTIL_FD_IS_VALID (in))
     grub_util_error (_("cannot open `%s': %s"), from, grub_util_fd_strerror ());
 
+  file_descs[idx].name = xstrdup (to);
+  file_descs[idx].source = xstrdup (from);
+  canonicalize (file_descs[idx].name);
+  file_descs[idx].mtime = grub_util_get_mtime (from);
+  file_descs[idx].size = grub_util_get_fd_size (in, from, NULL);
+
+  grub_util_fd_close (in);
+}
+
+static int
+filecmp (const void *p1, const void *p2)
+{
+  const struct file_desc *a = p1, *b = p2;
+
+  /* Don't use strcmp, it's buggy on some systems.  */
+  return grub_strcmp (a->name, b->name);
+}
+
+static void
+write_memdisk (char *memdisk_img)
+{
+  FILE *memdisk;
+  struct grub_greffs_header head;
+  struct grub_greffs_inode inode;
+  size_t total_strlen = 0, i;
+  size_t name_pad = 0;
+  grub_uint32_t file_offset;
+
+  qsort (file_descs, n_file_descs, sizeof (file_descs[0]), filecmp);
+
+  for (i = 0; i < n_file_descs; i++)
+    total_strlen += grub_strlen (file_descs[i].name);
+  name_pad = ALIGN_UP (total_strlen, 4) - total_strlen;
+  total_strlen += name_pad;
+
+  grub_memcpy (head.magic, GRUB_GREFFS_MAGIC, sizeof (head.magic));
+  head.nfiles = grub_cpu_to_le32 (n_file_descs);
+  head.inodes_offset = grub_cpu_to_le32 (sizeof (head)
+					 + sizeof (grub_uint32_t)
+					 * (n_file_descs + 1));
+  head.string_ptrs_offset = grub_cpu_to_le32 (sizeof (head));
+
+  memdisk = grub_util_fopen (memdisk_img, "wb");
+  if (!memdisk)
+    grub_util_error (_("Can't create file: %s"), strerror (errno));
+
+  fwrite (&head, 1, sizeof (head), memdisk);
+
+  grub_uint32_t curname = sizeof (head) + sizeof (grub_uint32_t)
+    * (n_file_descs + 1) + sizeof (inode) * n_file_descs;
+  for (i = 0; i <= n_file_descs; i++)
+    {
+      grub_uint32_t curname_le = grub_cpu_to_le32 (curname);
+      fwrite (&curname_le, 1, sizeof (curname_le), memdisk);
+      if (i != n_file_descs)
+	curname += grub_strlen (file_descs[i].name);
+    }
+
+  file_offset = sizeof (head) + sizeof (grub_uint32_t)
+    * (n_file_descs + 1) + sizeof (inode) * n_file_descs + total_strlen;
+  for (i = 0; i < n_file_descs; i++)
+    {
+      inode.start = grub_cpu_to_le32 (file_offset);
+      inode.size = grub_cpu_to_le32 (file_descs[i].size);
+      inode.mtime = grub_cpu_to_le32 (file_descs[i].mtime);
+      inode.type = 0;
+      fwrite (&inode, 1, sizeof (inode), memdisk);
+      file_offset += file_descs[i].size;
+    }
+
+  for (i = 0; i < n_file_descs; i++)
+    fwrite (file_descs[i].name, 1, grub_strlen (file_descs[i].name), memdisk);
+
   if (!grub_install_copy_buffer)
     grub_install_copy_buffer = xmalloc (GRUB_INSTALL_COPY_BUFFER_SIZE);
 
-  size = grub_util_get_fd_size (in, from, NULL);
+  grub_memset (grub_install_copy_buffer, 0, 4);
+  fwrite (grub_install_copy_buffer, 1, name_pad, memdisk);
 
-  memset (&hd, 0, sizeof (hd));
-  memcpy (hd.name, tcn, optr - tcn < 99 ? optr - tcn : 99);
-  memcpy (hd.mode, "0000600", 7);
-  memcpy (hd.uid, "0001750", 7);
-  memcpy (hd.gid, "0001750", 7);
-
-  set_tar_value (hd.size, size, 12);
-  set_tar_value (hd.mtime, mtime, 12);
-  hd.typeflag = '0';
-  memcpy (hd.magic, "ustar  ", 7);
-  memcpy (hd.uname, "grub", 4);
-  memcpy (hd.gname, "grub", 4);
-
-  compute_checksum (&hd);
-
-  fwrite (&hd, 1, sizeof (hd), memdisk);
- 
-  while (1)
+  for (i = 0; i < n_file_descs; i++)
     {
-      r = grub_util_fd_read (in, grub_install_copy_buffer, GRUB_INSTALL_COPY_BUFFER_SIZE);
-      if (r <= 0)
-	break;
-      fwrite (grub_install_copy_buffer, 1, r, memdisk);
-    }
-  grub_util_fd_close (in);
+      grub_util_fd_t in;
+      size_t remaining = file_descs[i].size;
+      in = grub_util_fd_open (file_descs[i].source, GRUB_UTIL_FD_O_RDONLY);
+      if (!GRUB_UTIL_FD_IS_VALID (in))
+	grub_util_error (_("cannot open `%s': %s"),
+			 file_descs[i].source, grub_util_fd_strerror ());
+ 
+      while (remaining)
+	{
+	  size_t toread = remaining;
+	  ssize_t r;
+	  if (remaining > GRUB_INSTALL_COPY_BUFFER_SIZE)
+	    toread = GRUB_INSTALL_COPY_BUFFER_SIZE;
+	  r = grub_util_fd_read (in, grub_install_copy_buffer, toread);
+	  if (r <= 0)
+	    break;
+	  fwrite (grub_install_copy_buffer, 1, r, memdisk);
+	  if (r >= remaining)
+	    remaining = 0;
+	  else
+	    remaining -= r; 
+	}
+      grub_util_fd_close (in);
 
-  write_pad (size);
+      grub_memset (grub_install_copy_buffer, 0, GRUB_INSTALL_COPY_BUFFER_SIZE);
+      while (remaining)
+	{
+	  size_t toread = remaining;
+	  if (remaining > GRUB_INSTALL_COPY_BUFFER_SIZE)
+	    toread = GRUB_INSTALL_COPY_BUFFER_SIZE;
+	  fwrite (grub_install_copy_buffer, 1, toread, memdisk);
+	  remaining -= toread; 
+	}
+    }
+
+  fclose (memdisk);
 }
 
 int
@@ -319,8 +353,6 @@ main (int argc, char *argv[])
 
   char *memdisk_img = grub_util_make_temporary_file ();
 
-  memdisk = grub_util_fopen (memdisk_img, "wb");
-
   add_tar_file (memdisk_dir, "");
   for (i = 0; i < nfiles; i++)
     {
@@ -341,14 +373,12 @@ main (int argc, char *argv[])
 	to++;
       add_tar_file (from, to);
     }
-  write_zeros (512);
-
-  fclose (memdisk);
 
+  write_memdisk (memdisk_img);
   grub_util_unlink_recursive (memdisk_dir);
 
   grub_install_push_module ("memdisk");
-  grub_install_push_module ("tar");
+  grub_install_push_module ("greffs");
 
   grub_install_make_image_wrap (grub_install_source_directory,
 				"(memdisk)/boot/grub", output_image,
@@ -356,5 +386,6 @@ main (int argc, char *argv[])
 				grub_util_get_target_name (format), 0);
 
   grub_util_unlink (memdisk_img);
+
   return 0;
 }