copy_file_range: New function to copy file data

The semantics are based on the Linux system call, but a very close
emulation in user space is provided.

6 files changed, 1056 insertions, 2 deletions

diff --git a/io/Makefile b/io/Makefile
index c72519541f..85eb927d4c 100644
--- a/io/Makefile
+++ b/io/Makefile
@@ -52,7 +52,7 @@ routines :=								\
 	ftw ftw64 fts fts64 poll ppoll					\
 	posix_fadvise posix_fadvise64					\
 	posix_fallocate posix_fallocate64				\
-	sendfile sendfile64 \
+	sendfile sendfile64 copy_file_range 				\
 	utimensat futimens
 
 # These routines will be omitted from the libc shared object.
@@ -70,7 +70,13 @@ tests		:= test-utime test-stat test-stat2 test-lfs tst-getcwd \
 		   tst-symlinkat tst-linkat tst-readlinkat tst-mkdirat \
 		   tst-mknodat tst-mkfifoat tst-ttyname_r bug-ftw5 \
 		   tst-posix_fallocate tst-posix_fallocate64 \
-		   tst-fts tst-fts-lfs tst-open-tmpfile
+		   tst-fts tst-fts-lfs tst-open-tmpfile \
+		   tst-copy_file_range \
+
+# This test includes the compat implementation of copy_file_range,
+# which uses internal, unexported libc functions.
+tests-static += tst-copy_file_range-compat
+tests-internal += tst-copy_file_range-compat
 
 ifeq ($(run-built-tests),yes)
 tests-special += $(objpfx)ftwtest.out
diff --git a/io/Versions b/io/Versions
index 64316cd025..98898cb9d5 100644
--- a/io/Versions
+++ b/io/Versions
@@ -125,4 +125,7 @@ libc {
   GLIBC_2.23 {
     fts64_children; fts64_close; fts64_open; fts64_read; fts64_set;
   }
+  GLIBC_2.27 {
+    copy_file_range;
+  }
 }
diff --git a/io/copy_file_range-compat.c b/io/copy_file_range-compat.c
new file mode 100644
index 0000000000..5c1b7b3258
--- /dev/null
+++ b/io/copy_file_range-compat.c
@@ -0,0 +1,160 @@
+/* Emulation of copy_file_range.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* The following macros should be defined before including this
+   file:
+
+   COPY_FILE_RANGE_DECL   Declaration specifiers for the function below.
+   COPY_FILE_RANGE        Name of the function to define.  */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+COPY_FILE_RANGE_DECL
+ssize_t
+COPY_FILE_RANGE (int infd, __off64_t *pinoff,
+                 int outfd, __off64_t *poutoff,
+                 size_t length, unsigned int flags)
+{
+  if (flags != 0)
+    {
+      __set_errno (EINVAL);
+      return -1;
+    }
+
+  {
+    struct stat64 instat;
+    struct stat64 outstat;
+    if (fstat64 (infd, &instat) != 0 || fstat64 (outfd, &outstat) != 0)
+      return -1;
+    if (S_ISDIR (instat.st_mode) || S_ISDIR (outstat.st_mode))
+      {
+        __set_errno (EISDIR);
+        return -1;
+      }
+    if (!S_ISREG (instat.st_mode) || !S_ISREG (outstat.st_mode))
+      {
+        /* We need a regular input file so that the we can seek
+           backwards in case of a write failure.  */
+        __set_errno (EINVAL);
+        return -1;
+      }
+    if (instat.st_dev != outstat.st_dev)
+      {
+        /* Cross-device copies are not supported.  */
+        __set_errno (EXDEV);
+        return -1;
+      }
+  }
+
+  /* The output descriptor must not have O_APPEND set.  */
+  {
+    int flags = __fcntl (outfd, F_GETFL);
+    if (flags & O_APPEND)
+      {
+        __set_errno (EBADF);
+        return -1;
+      }
+  }
+
+  /* Avoid an overflow in the result.  */
+  if (length > SSIZE_MAX)
+    length = SSIZE_MAX;
+
+  /* Main copying loop.  The buffer size is arbitrary and is a
+     trade-off between stack size consumption, cache usage, and
+     amortization of system call overhead.  */
+  size_t copied = 0;
+  char buf[8192];
+  while (length > 0)
+    {
+      size_t to_read = length;
+      if (to_read > sizeof (buf))
+        to_read = sizeof (buf);
+
+      /* Fill the buffer.  */
+      ssize_t read_count;
+      if (pinoff == NULL)
+        read_count = read (infd, buf, to_read);
+      else
+        read_count = __libc_pread64 (infd, buf, to_read, *pinoff);
+      if (read_count == 0)
+        /* End of file reached prematurely.  */
+        return copied;
+      if (read_count < 0)
+        {
+          if (copied > 0)
+            /* Report the number of bytes copied so far.  */
+            return copied;
+          return -1;
+        }
+      if (pinoff != NULL)
+        *pinoff += read_count;
+
+      /* Write the buffer part which was read to the destination.  */
+      char *end = buf + read_count;
+      for (char *p = buf; p < end; )
+        {
+          ssize_t write_count;
+          if (poutoff == NULL)
+            write_count = write (outfd, p, end - p);
+          else
+            write_count = __libc_pwrite64 (outfd, p, end - p, *poutoff);
+          if (write_count < 0)
+            {
+              /* Adjust the input read position to match what we have
+                 written, so that the caller can pick up after the
+                 error.  */
+              size_t written = p - buf;
+              /* NB: This needs to be signed so that we can form the
+                 negative value below.  */
+              ssize_t overread = read_count - written;
+              if (pinoff == NULL)
+                {
+                  if (overread > 0)
+                    {
+                      /* We are on an error recovery path, so we
+                         cannot deal with failure here.  */
+                      int save_errno = errno;
+                      (void) __libc_lseek64 (infd, -overread, SEEK_CUR);
+                      __set_errno (save_errno);
+                    }
+                }
+              else /* pinoff != NULL */
+                *pinoff -= overread;
+
+              if (copied + written > 0)
+                /* Report the number of bytes copied so far.  */
+                return copied + written;
+              return -1;
+            }
+          p += write_count;
+          if (poutoff != NULL)
+            *poutoff += write_count;
+        } /* Write loop.  */
+
+      copied += read_count;
+      length -= read_count;
+    }
+  return copied;
+}
diff --git a/io/copy_file_range.c b/io/copy_file_range.c
new file mode 100644
index 0000000000..61ee6871b4
--- /dev/null
+++ b/io/copy_file_range.c
@@ -0,0 +1,22 @@
+/* Generic implementation of copy_file_range.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#define COPY_FILE_RANGE_DECL
+#define COPY_FILE_RANGE copy_file_range
+
+#include <io/copy_file_range-compat.c>
diff --git a/io/tst-copy_file_range-compat.c b/io/tst-copy_file_range-compat.c
new file mode 100644
index 0000000000..eb737946d9
--- /dev/null
+++ b/io/tst-copy_file_range-compat.c
@@ -0,0 +1,30 @@
+/* Test the fallback implementation of copy_file_range.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Get the declaration of the official copy_of_range function.  */
+#include <unistd.h>
+
+/* Compile a local version of copy_file_range.  */
+#define COPY_FILE_RANGE_DECL static
+#define COPY_FILE_RANGE copy_file_range_compat
+#include <io/copy_file_range-compat.c>
+
+/* Re-use the test, but run it against copy_file_range_compat defined
+   above.  */
+#define copy_file_range copy_file_range_compat
+#include "tst-copy_file_range.c"
diff --git a/io/tst-copy_file_range.c b/io/tst-copy_file_range.c
new file mode 100644
index 0000000000..d8f4e8ac04
--- /dev/null
+++ b/io/tst-copy_file_range.c
@@ -0,0 +1,833 @@
+/* Tests for copy_file_range.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <array_length.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <libgen.h>
+#include <poll.h>
+#include <sched.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <support/check.h>
+#include <support/namespace.h>
+#include <support/support.h>
+#include <support/temp_file.h>
+#include <support/test-driver.h>
+#include <support/xunistd.h>
+#include <sys/mount.h>
+
+/* Boolean flags which indicate whether to use pointers with explicit
+   output flags.  */
+static int do_inoff;
+static int do_outoff;
+
+/* Name and descriptors of the input files.  Files are truncated and
+   reopened (with O_RDWR) between tests.  */
+static char *infile;
+static int infd;
+static char *outfile;
+static int outfd;
+
+/* Like the above, but on a different file system.  xdevfile can be
+   NULL if no suitable file system has been found.  */
+static char *xdevfile;
+
+/* Input and output offsets.  Set according to do_inoff and do_outoff
+   before the test.  The offsets themselves are always set to
+   zero.  */
+static off64_t inoff;
+static off64_t *pinoff;
+static off64_t outoff;
+static off64_t *poutoff;
+
+/* These are a collection of copy sizes used in tests.  The selection
+   takes into account that the fallback implementation uses an
+   internal buffer of 8192 bytes.  */
+enum { maximum_size = 99999 };
+static const int typical_sizes[] =
+  { 0, 1, 2, 3, 1024, 2048, 4096, 8191, 8192, 8193, 16383, 16384, 16385,
+    maximum_size };
+
+/* The random contents of this array can be used as a pattern to check
+   for correct write operations.  */
+static unsigned char random_data[maximum_size];
+
+/* The size chosen by the test harness.  */
+static int current_size;
+
+/* Maximum writable file offset.  Updated by find_maximum_offset
+   below.  */
+static off64_t maximum_offset;
+
+/* Error code when crossing the offset.  */
+static int maximum_offset_errno;
+
+/* If true: Writes which cross the limit will fail.  If false: Writes
+   which cross the limit will result in a partial write.  */
+static bool maximum_offset_hard_limit;
+
+/* Fills maximum_offset etc. above.  Truncates outfd as a side
+   effect.  */
+static void
+find_maximum_offset (void)
+{
+  xftruncate (outfd, 0);
+  if (maximum_offset != 0)
+    return;
+
+  uint64_t upper = -1;
+  upper >>= 1;                  /* Maximum of off64_t.  */
+  TEST_VERIFY ((off64_t) upper > 0);
+  TEST_VERIFY ((off64_t) (upper + 1) < 0);
+  if (lseek64 (outfd, upper, SEEK_SET) >= 0)
+    {
+      if (write (outfd, "", 1) == 1)
+        FAIL_EXIT1 ("created a file larger than the off64_t range");
+    }
+
+  uint64_t lower = 1024 * 1024; /* A reasonable minimum file size.  */
+  /* Loop invariant: writing at lower succeeds, writing at upper fails.  */
+  while (lower + 1 < upper)
+    {
+      uint64_t middle = (lower + upper) / 2;
+      if (test_verbose > 0)
+        printf ("info: %s: remaining test range %" PRIu64 " .. %" PRIu64
+                ", probe at %" PRIu64 "\n", __func__, lower, upper, middle);
+      xftruncate (outfd, 0);
+      if (lseek64 (outfd, middle, SEEK_SET) >= 0
+          && write (outfd, "", 1) == 1)
+        lower = middle;
+      else
+        upper = middle;
+    }
+  TEST_VERIFY (lower + 1 == upper);
+  maximum_offset = lower;
+  printf ("info: maximum writable file offset: %" PRIu64 " (%" PRIx64 ")\n",
+          lower, lower);
+
+  /* Check that writing at the valid offset actually works.  */
+  xftruncate (outfd, 0);
+  xlseek (outfd, lower, SEEK_SET);
+  TEST_COMPARE (write (outfd, "", 1), 1);
+
+  /* Cross the boundary with a two-byte write.  This can either result
+     in a short write, or a failure.  */
+  xlseek (outfd, lower, SEEK_SET);
+  ssize_t ret = write (outfd, " ", 2);
+  if (ret < 0)
+    {
+      maximum_offset_errno = errno;
+      maximum_offset_hard_limit = true;
+    }
+  else
+    maximum_offset_hard_limit = false;
+
+  /* Check that writing at the next offset actually fails.  This also
+     obtains the expected errno value.  */
+  xftruncate (outfd, 0);
+  const char *action;
+  if (lseek64 (outfd, lower + 1, SEEK_SET) != 0)
+    {
+      if (write (outfd, "", 1) != -1)
+        FAIL_EXIT1 ("write to impossible offset %" PRIu64 " succeeded",
+                    lower + 1);
+      action = "writing";
+      int errno_copy = errno;
+      if (maximum_offset_hard_limit)
+        TEST_COMPARE (errno_copy, maximum_offset_errno);
+      else
+        maximum_offset_errno = errno_copy;
+    }
+  else
+    {
+      action = "seeking";
+      maximum_offset_errno = errno;
+    }
+  printf ("info: %s out of range fails with %m (%d)\n",
+          action, maximum_offset_errno);
+
+  xftruncate (outfd, 0);
+  xlseek (outfd, 0, SEEK_SET);
+}
+
+/* Perform a copy of a file.  */
+static void
+simple_file_copy (void)
+{
+  xwrite (infd, random_data, current_size);
+
+  int length;
+  int in_skipped; /* Expected skipped bytes in input.  */
+  if (do_inoff)
+    {
+      xlseek (infd, 1, SEEK_SET);
+      inoff = 2;
+      length = current_size - 3;
+      in_skipped = 2;
+    }
+  else
+    {
+      xlseek (infd, 3, SEEK_SET);
+      length = current_size - 5;
+      in_skipped = 3;
+    }
+  int out_skipped; /* Expected skipped bytes before the written data.  */
+  if (do_outoff)
+    {
+      xlseek (outfd, 4, SEEK_SET);
+      outoff = 5;
+      out_skipped = 5;
+    }
+  else
+    {
+      xlseek (outfd, 6, SEEK_SET);
+      length = current_size - 6;
+      out_skipped = 6;
+    }
+  if (length < 0)
+    length = 0;
+
+  TEST_COMPARE (copy_file_range (infd, pinoff, outfd, poutoff,
+                                 length, 0), length);
+  if (do_inoff)
+    {
+      TEST_COMPARE (inoff, 2 + length);
+      TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 1);
+    }
+  else
+    TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 3 + length);
+  if (do_outoff)
+    {
+      TEST_COMPARE (outoff, 5 + length);
+      TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 4);
+    }
+  else
+    TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 6 + length);
+
+  struct stat64 st;
+  xfstat (outfd, &st);
+  if (length > 0)
+    TEST_COMPARE (st.st_size, out_skipped + length);
+  else
+    {
+      /* If we did not write anything, we also did not add any
+         padding.  */
+      TEST_COMPARE (st.st_size, 0);
+      return;
+    }
+
+  xlseek (outfd, 0, SEEK_SET);
+  char *bytes = xmalloc (st.st_size);
+  TEST_COMPARE (read (outfd, bytes, st.st_size), st.st_size);
+  for (int i = 0; i < out_skipped; ++i)
+    TEST_COMPARE (bytes[i], 0);
+  TEST_VERIFY (memcmp (bytes + out_skipped, random_data + in_skipped,
+                       length) == 0);
+  free (bytes);
+}
+
+/* Test that reading from a pipe willfails.  */
+static void
+pipe_as_source (void)
+{
+  int pipefds[2];
+  xpipe (pipefds);
+
+  for (int length = 0; length < 2; ++length)
+    {
+      if (test_verbose > 0)
+        printf ("info: %s: length=%d\n", __func__, length);
+
+      /* Make sure that there is something to copy in the pipe.  */
+      xwrite (pipefds[1], "@", 1);
+
+      TEST_COMPARE (copy_file_range (pipefds[0], pinoff, outfd, poutoff,
+                                     length, 0), -1);
+      /* Linux 4.10 and later return EINVAL.  Older kernels return
+         EXDEV.  */
+      TEST_VERIFY (errno == EINVAL || errno == EXDEV);
+      TEST_COMPARE (inoff, 0);
+      TEST_COMPARE (outoff, 0);
+      TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 0);
+
+      /* Make sure that nothing was read.  */
+      char buf = 'A';
+      TEST_COMPARE (read (pipefds[0], &buf, 1), 1);
+      TEST_COMPARE (buf, '@');
+    }
+
+  xclose (pipefds[0]);
+  xclose (pipefds[1]);
+}
+
+/* Test that writing to a pipe fails.  */
+static void
+pipe_as_destination (void)
+{
+  /* Make sure that there is something to read in the input file.  */
+  xwrite (infd, "abc", 3);
+  xlseek (infd, 0, SEEK_SET);
+
+  int pipefds[2];
+  xpipe (pipefds);
+
+  for (int length = 0; length < 2; ++length)
+    {
+      if (test_verbose > 0)
+        printf ("info: %s: length=%d\n", __func__, length);
+
+      TEST_COMPARE (copy_file_range (infd, pinoff, pipefds[1], poutoff,
+                                     length, 0), -1);
+      /* Linux 4.10 and later return EINVAL.  Older kernels return
+         EXDEV.  */
+      TEST_VERIFY (errno == EINVAL || errno == EXDEV);
+      TEST_COMPARE (inoff, 0);
+      TEST_COMPARE (outoff, 0);
+      TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0);
+
+      /* Make sure that nothing was written.  */
+      struct pollfd pollfd = { .fd = pipefds[0], .events = POLLIN, };
+      TEST_COMPARE (poll (&pollfd, 1, 0), 0);
+    }
+
+  xclose (pipefds[0]);
+  xclose (pipefds[1]);
+}
+
+/* Test a write failure after (potentially) writing some bytes.
+   Failure occurs near the start of the buffer.  */
+static void
+delayed_write_failure_beginning (void)
+{
+  /* We need to write something to provoke the error.  */
+  if (current_size == 0)
+    return;
+  xwrite (infd, random_data, sizeof (random_data));
+  xlseek (infd, 0, SEEK_SET);
+
+  /* Write failure near the start.  The actual error code varies among
+     file systems.  */
+  find_maximum_offset ();
+  off64_t where = maximum_offset;
+
+  if (current_size == 1)
+    ++where;
+  outoff = where;
+  if (do_outoff)
+    xlseek (outfd, 1, SEEK_SET);
+  else
+    xlseek (outfd, where, SEEK_SET);
+  if (maximum_offset_hard_limit || where > maximum_offset)
+    {
+      TEST_COMPARE (copy_file_range (infd, pinoff, outfd, poutoff,
+                                     sizeof (random_data), 0), -1);
+      TEST_COMPARE (errno, maximum_offset_errno);
+      TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0);
+      TEST_COMPARE (inoff, 0);
+      if (do_outoff)
+        TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 1);
+      else
+        TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), where);
+      TEST_COMPARE (outoff, where);
+      struct stat64 st;
+      xfstat (outfd, &st);
+      TEST_COMPARE (st.st_size, 0);
+    }
+  else
+    {
+      /* The offset is not a hard limit.  This means we write one
+         byte.  */
+      TEST_COMPARE (copy_file_range (infd, pinoff, outfd, poutoff,
+                                     sizeof (random_data), 0), 1);
+      if (do_inoff)
+        {
+          TEST_COMPARE (inoff, 1);
+          TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0);
+        }
+      else
+        {
+          TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 1);
+          TEST_COMPARE (inoff, 0);
+        }
+      if (do_outoff)
+        {
+          TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 1);
+          TEST_COMPARE (outoff, where + 1);
+        }
+      else
+        {
+          TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), where + 1);
+          TEST_COMPARE (outoff, where);
+        }
+      struct stat64 st;
+      xfstat (outfd, &st);
+      TEST_COMPARE (st.st_size, where + 1);
+    }
+}
+
+/* Test a write failure after (potentially) wr