00001 #define _LARGEFILE64_SOURCE
00002 #define _FILE_OFFSET_BITS 64
00003
00004 #include "util/file.hh"
00005
00006 #include "util/exception.hh"
00007
00008 #include <algorithm>
00009 #include <cstdlib>
00010 #include <cstdio>
00011 #include <iostream>
00012 #include <limits>
00013 #include <sstream>
00014
00015
00016 #include <cassert>
00017 #include <cerrno>
00018 #include <climits>
00019 #include <sys/types.h>
00020 #include <sys/stat.h>
00021 #include <fcntl.h>
00022 #include <stdint.h>
00023
00024 #if defined(__MINGW32__)
00025 #include <windows.h>
00026 #include <unistd.h>
00027 #warning "The file functions on MinGW have not been tested for file sizes above 2^31 - 1. Please read https://stackoverflow.com/questions/12539488/determine-64-bit-file-size-in-c-on-mingw-32-bit and fix"
00028 #elif defined(_WIN32) || defined(_WIN64)
00029 #include <windows.h>
00030 #include <io.h>
00031 #else
00032 #include <unistd.h>
00033 #endif
00034
00035 namespace util {
00036
00037 scoped_fd::~scoped_fd() {
00038 if (fd_ != -1 && close(fd_)) {
00039 std::cerr << "Could not close file " << fd_ << std::endl;
00040 std::abort();
00041 }
00042 }
00043
00044 void scoped_FILE_closer::Close(std::FILE *file) {
00045 if (file && std::fclose(file)) {
00046 std::cerr << "Could not close file " << file << std::endl;
00047 std::abort();
00048 }
00049 }
00050
00051
00052 FDException::FDException(int fd) throw() : fd_(fd), name_guess_(NameFromFD(fd)) {
00053 *this << "in " << name_guess_ << ' ';
00054 }
00055
00056 FDException::~FDException() throw() {}
00057
00058 EndOfFileException::EndOfFileException() throw() {
00059 *this << "End of file";
00060 }
00061 EndOfFileException::~EndOfFileException() throw() {}
00062
00063 bool InputFileIsStdin(StringPiece path) {
00064 return path == "-" || path == "/dev/stdin";
00065 }
00066
00067 bool OutputFileIsStdout(StringPiece path) {
00068 return path == "-" || path == "/dev/stdout";
00069 }
00070
00071 int OpenReadOrThrow(const char *name) {
00072 int ret;
00073 #if defined(_WIN32) || defined(_WIN64)
00074 UTIL_THROW_IF(-1 == (ret = _open(name, _O_BINARY | _O_RDONLY)), ErrnoException, "while opening " << name);
00075 #else
00076 UTIL_THROW_IF(-1 == (ret = open(name, O_RDONLY)), ErrnoException, "while opening " << name);
00077 #endif
00078 return ret;
00079 }
00080
00081 int CreateOrThrow(const char *name) {
00082 int ret;
00083 #if defined(_WIN32) || defined(_WIN64)
00084 UTIL_THROW_IF(-1 == (ret = _open(name, _O_CREAT | _O_TRUNC | _O_RDWR | _O_BINARY, _S_IREAD | _S_IWRITE)), ErrnoException, "while creating " << name);
00085 #else
00086 UTIL_THROW_IF(-1 == (ret = open(name, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)), ErrnoException, "while creating " << name);
00087 #endif
00088 return ret;
00089 }
00090
00091 uint64_t SizeFile(int fd) {
00092 #if defined __MINGW32__
00093 struct stat sb;
00094
00095 int ret = fstat(fd, &sb);
00096 if (ret == -1 || (!sb.st_size && !S_ISREG(sb.st_mode))) return kBadSize;
00097 return sb.st_size;
00098 #elif defined(_WIN32) || defined(_WIN64)
00099 __int64 ret = _filelengthi64(fd);
00100 return (ret == -1) ? kBadSize : ret;
00101 #else // Not windows.
00102
00103 #ifdef OS_ANDROID
00104 struct stat64 sb;
00105 int ret = fstat64(fd, &sb);
00106 #else
00107 struct stat sb;
00108 int ret = fstat(fd, &sb);
00109 #endif
00110 if (ret == -1 || (!sb.st_size && !S_ISREG(sb.st_mode))) return kBadSize;
00111 return sb.st_size;
00112 #endif
00113 }
00114
00115 uint64_t SizeOrThrow(int fd) {
00116 uint64_t ret = SizeFile(fd);
00117 UTIL_THROW_IF_ARG(ret == kBadSize, FDException, (fd), "Failed to size");
00118 return ret;
00119 }
00120
00121 void ResizeOrThrow(int fd, uint64_t to) {
00122 #if defined __MINGW32__
00123
00124 int ret = ftruncate
00125 #elif defined(_WIN32) || defined(_WIN64)
00126 errno_t ret = _chsize_s
00127 #elif defined(OS_ANDROID)
00128 int ret = ftruncate64
00129 #else
00130 int ret = ftruncate
00131 #endif
00132 (fd, to);
00133 UTIL_THROW_IF_ARG(ret, FDException, (fd), "while resizing to " << to << " bytes");
00134 }
00135
00136 namespace {
00137 std::size_t GuardLarge(std::size_t size) {
00138
00139
00140
00141
00142 #if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) || defined(OS_ANDROID) || defined(__MINGW32__)
00143 return size < INT_MAX ? size : INT_MAX;
00144 #else
00145 return size;
00146 #endif
00147 }
00148 }
00149
00150 #if defined(_WIN32) || defined(_WIN64)
00151 namespace {
00152 const std::size_t kMaxDWORD = static_cast<std::size_t>(4294967295UL);
00153 }
00154 #endif
00155
00156 std::size_t PartialRead(int fd, void *to, std::size_t amount) {
00157 #if defined(_WIN32) || defined(_WIN64)
00158 DWORD ret;
00159 HANDLE file_handle = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
00160 DWORD larger_size = static_cast<DWORD>(std::min<std::size_t>(kMaxDWORD, amount));
00161 DWORD smaller_size = 28672;
00162 if (!ReadFile(file_handle, to, larger_size, &ret, NULL))
00163 {
00164 DWORD last_error = GetLastError();
00165 if (last_error != ERROR_NOT_ENOUGH_MEMORY || !ReadFile(file_handle, to, smaller_size, &ret, NULL)) {
00166 UTIL_THROW(WindowsException, "Windows error in ReadFile.");
00167 }
00168 }
00169 #else
00170 errno = 0;
00171 ssize_t ret;
00172 do {
00173 ret = read(fd, to, GuardLarge(amount));
00174 } while (ret == -1 && errno == EINTR);
00175 UTIL_THROW_IF_ARG(ret < 0, FDException, (fd), "while reading " << amount << " bytes");
00176 #endif
00177 return static_cast<std::size_t>(ret);
00178 }
00179
00180 void ReadOrThrow(int fd, void *to_void, std::size_t amount) {
00181 uint8_t *to = static_cast<uint8_t*>(to_void);
00182 while (amount) {
00183 std::size_t ret = PartialRead(fd, to, amount);
00184 UTIL_THROW_IF(ret == 0, EndOfFileException, " in " << NameFromFD(fd) << " but there should be " << amount << " more bytes to read.");
00185 amount -= ret;
00186 to += ret;
00187 }
00188 }
00189
00190 std::size_t ReadOrEOF(int fd, void *to_void, std::size_t amount) {
00191 uint8_t *to = static_cast<uint8_t*>(to_void);
00192 std::size_t remaining = amount;
00193 while (remaining) {
00194 std::size_t ret = PartialRead(fd, to, remaining);
00195 if (!ret) return amount - remaining;
00196 remaining -= ret;
00197 to += ret;
00198 }
00199 return amount;
00200 }
00201
00202 void WriteOrThrow(int fd, const void *data_void, std::size_t size) {
00203 const uint8_t *data = static_cast<const uint8_t*>(data_void);
00204 while (size) {
00205 #if defined(_WIN32) || defined(_WIN64)
00206 int ret;
00207 #else
00208 ssize_t ret;
00209 #endif
00210 errno = 0;
00211 do {
00212 ret =
00213 #if defined(_WIN32) || defined(_WIN64)
00214 _write
00215 #else
00216 write
00217 #endif
00218 (fd, data, GuardLarge(size));
00219 } while (ret == -1 && errno == EINTR);
00220 UTIL_THROW_IF_ARG(ret < 1, FDException, (fd), "while writing " << size << " bytes");
00221 data += ret;
00222 size -= ret;
00223 }
00224 }
00225
00226 void WriteOrThrow(FILE *to, const void *data, std::size_t size) {
00227 if (!size) return;
00228 UTIL_THROW_IF(1 != std::fwrite(data, size, 1, to), ErrnoException, "Short write; requested size " << size);
00229 }
00230
00231 void ErsatzPRead(int fd, void *to_void, std::size_t size, uint64_t off) {
00232 uint8_t *to = static_cast<uint8_t*>(to_void);
00233 while (size) {
00234 #if defined(_WIN32) || defined(_WIN64)
00235
00236
00237 DWORD reading = static_cast<DWORD>(std::min<std::size_t>(kMaxDWORD, size));
00238 DWORD ret;
00239 OVERLAPPED overlapped;
00240 memset(&overlapped, 0, sizeof(OVERLAPPED));
00241 overlapped.Offset = static_cast<DWORD>(off);
00242 overlapped.OffsetHigh = static_cast<DWORD>(off >> 32);
00243 UTIL_THROW_IF(!ReadFile((HANDLE)_get_osfhandle(fd), to, reading, &ret, &overlapped), WindowsException, "ReadFile failed for offset " << off);
00244 #else
00245 ssize_t ret;
00246 errno = 0;
00247 ret =
00248 #ifdef OS_ANDROID
00249 pread64
00250 #else
00251 pread
00252 #endif
00253 (fd, to, GuardLarge(size), off);
00254 if (ret <= 0) {
00255 if (ret == -1 && errno == EINTR) continue;
00256 UTIL_THROW_IF(ret == 0, EndOfFileException, " for reading " << size << " bytes at " << off << " from " << NameFromFD(fd));
00257 UTIL_THROW_ARG(FDException, (fd), "while reading " << size << " bytes at offset " << off);
00258 }
00259 #endif
00260 size -= ret;
00261 off += ret;
00262 to += ret;
00263 }
00264 }
00265
00266 void ErsatzPWrite(int fd, const void *from_void, std::size_t size, uint64_t off) {
00267 const uint8_t *from = static_cast<const uint8_t*>(from_void);
00268 while(size) {
00269 #if defined(_WIN32) || defined(_WIN64)
00270
00271
00272 DWORD writing = static_cast<DWORD>(std::min<std::size_t>(kMaxDWORD, size));
00273 DWORD ret;
00274 OVERLAPPED overlapped;
00275 memset(&overlapped, 0, sizeof(OVERLAPPED));
00276 overlapped.Offset = static_cast<DWORD>(off);
00277 overlapped.OffsetHigh = static_cast<DWORD>(off >> 32);
00278 UTIL_THROW_IF(!WriteFile((HANDLE)_get_osfhandle(fd), from, writing, &ret, &overlapped), Exception, "WriteFile failed for offset " << off);
00279 #else
00280 ssize_t ret;
00281 errno = 0;
00282 ret =
00283 #ifdef OS_ANDROID
00284 pwrite64
00285 #else
00286 pwrite
00287 #endif
00288 (fd, from, GuardLarge(size), off);
00289 if (ret <= 0) {
00290 if (ret == -1 && errno == EINTR) continue;
00291 UTIL_THROW_IF(ret == 0, EndOfFileException, " for writing " << size << " bytes at " << off << " from " << NameFromFD(fd));
00292 UTIL_THROW_ARG(FDException, (fd), "while writing " << size << " bytes at offset " << off);
00293 }
00294 #endif
00295 size -= ret;
00296 off += ret;
00297 from += ret;
00298 }
00299 }
00300
00301
00302 void FSyncOrThrow(int fd) {
00303
00304 #if !defined(_WIN32) && !defined(_WIN64)
00305 UTIL_THROW_IF_ARG(-1 == fsync(fd), FDException, (fd), "while syncing");
00306 #endif
00307 }
00308
00309 namespace {
00310
00311
00312 #if !defined(_WIN32) && !defined(_WIN64) && !defined(OS_ANDROID)
00313 template <unsigned> struct CheckOffT;
00314 template <> struct CheckOffT<8> {
00315 struct True {};
00316 };
00317
00318
00319 typedef CheckOffT<sizeof(off_t)>::True IgnoredType;
00320 #endif
00321
00322
00323 void InternalSeek(int fd, int64_t off, int whence) {
00324 if (
00325 #if defined __MINGW32__
00326
00327 (off_t)-1 == lseek(fd, off, whence)
00328 #elif defined(_WIN32) || defined(_WIN64)
00329 (__int64)-1 == _lseeki64(fd, off, whence)
00330 #elif defined(OS_ANDROID)
00331 (off64_t)-1 == lseek64(fd, off, whence)
00332 #else
00333 (off_t)-1 == lseek(fd, off, whence)
00334 #endif
00335 ) UTIL_THROW_ARG(FDException, (fd), "while seeking to " << off << " whence " << whence);
00336 }
00337 }
00338
00339 void SeekOrThrow(int fd, uint64_t off) {
00340 InternalSeek(fd, off, SEEK_SET);
00341 }
00342
00343 void AdvanceOrThrow(int fd, int64_t off) {
00344 InternalSeek(fd, off, SEEK_CUR);
00345 }
00346
00347 void SeekEnd(int fd) {
00348 InternalSeek(fd, 0, SEEK_END);
00349 }
00350
00351 std::FILE *FDOpenOrThrow(scoped_fd &file) {
00352 std::FILE *ret = fdopen(file.get(), "r+b");
00353 UTIL_THROW_IF_ARG(!ret, FDException, (file.get()), "Could not fdopen for write");
00354 file.release();
00355 return ret;
00356 }
00357
00358 std::FILE *FDOpenReadOrThrow(scoped_fd &file) {
00359 std::FILE *ret = fdopen(file.get(), "rb");
00360 UTIL_THROW_IF_ARG(!ret, FDException, (file.get()), "Could not fdopen for read");
00361 file.release();
00362 return ret;
00363 }
00364
00365
00366 #if defined(_WIN32) || defined(_WIN64)
00367
00368
00369
00370
00371
00372
00373
00374
00375
00376
00377
00378 static const char letters[] =
00379 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
00380
00381
00382
00383
00384
00385 int
00386 mkstemp_and_unlink(char *tmpl)
00387 {
00388 int len;
00389 char *XXXXXX;
00390 static unsigned long long value;
00391 unsigned long long random_time_bits;
00392 unsigned int count;
00393 int fd = -1;
00394 int save_errno = errno;
00395
00396
00397
00398
00399
00400
00401
00402 #define ATTEMPTS_MIN (62 * 62 * 62)
00403
00404
00405
00406 #if ATTEMPTS_MIN < TMP_MAX
00407 unsigned int attempts = TMP_MAX;
00408 #else
00409 unsigned int attempts = ATTEMPTS_MIN;
00410 #endif
00411
00412 len = strlen (tmpl);
00413 if (len < 6 || strcmp (&tmpl[len - 6], "XXXXXX"))
00414 {
00415 errno = EINVAL;
00416 return -1;
00417 }
00418
00419
00420 XXXXXX = &tmpl[len - 6];
00421
00422
00423 {
00424 SYSTEMTIME stNow;
00425 FILETIME ftNow;
00426
00427
00428 GetSystemTime(&stNow);
00429 stNow.wMilliseconds = 500;
00430 if (!SystemTimeToFileTime(&stNow, &ftNow))
00431 {
00432 errno = -1;
00433 return -1;
00434 }
00435
00436 random_time_bits = (((unsigned long long)ftNow.dwHighDateTime << 32)
00437 | (unsigned long long)ftNow.dwLowDateTime);
00438 }
00439 value += random_time_bits ^ (unsigned long long)GetCurrentThreadId ();
00440
00441 for (count = 0; count < attempts; value += 7777, ++count)
00442 {
00443 unsigned long long v = value;
00444
00445
00446 XXXXXX[0] = letters[v % 62];
00447 v /= 62;
00448 XXXXXX[1] = letters[v % 62];
00449 v /= 62;
00450 XXXXXX[2] = letters[v % 62];
00451 v /= 62;
00452 XXXXXX[3] = letters[v % 62];
00453 v /= 62;
00454 XXXXXX[4] = letters[v % 62];
00455 v /= 62;
00456 XXXXXX[5] = letters[v % 62];
00457
00458
00459
00460 int flags = _O_RDWR | _O_CREAT | _O_EXCL | _O_BINARY;
00461 flags |= _O_TEMPORARY;
00462 fd = _open (tmpl, flags, _S_IREAD | _S_IWRITE);
00463 if (fd >= 0)
00464 {
00465 errno = save_errno;
00466 return fd;
00467 }
00468 else if (errno != EEXIST)
00469 return -1;
00470 }
00471
00472
00473 errno = EEXIST;
00474 return -1;
00475 }
00476 #else
00477 int
00478 mkstemp_and_unlink(char *tmpl) {
00479 int ret = mkstemp(tmpl);
00480 if (ret != -1) {
00481 UTIL_THROW_IF(unlink(tmpl), ErrnoException, "while deleting delete " << tmpl);
00482 }
00483 return ret;
00484 }
00485 #endif
00486
00487
00488
00489 void NormalizeTempPrefix(std::string &base) {
00490 if (base.empty()) return;
00491 if (base[base.size() - 1] == '/') return;
00492 struct stat sb;
00493
00494 if (-1 == stat(base.c_str(), &sb)) return;
00495 if (
00496 #if defined(_WIN32) || defined(_WIN64)
00497 sb.st_mode & _S_IFDIR
00498 #else
00499 S_ISDIR(sb.st_mode)
00500 #endif
00501 ) base += '/';
00502 }
00503
00504 int MakeTemp(const StringPiece &base) {
00505 std::string name(base.data(), base.size());
00506 name += "XXXXXX";
00507 name.push_back(0);
00508 int ret;
00509 UTIL_THROW_IF(-1 == (ret = mkstemp_and_unlink(&name[0])), ErrnoException, "while making a temporary based on " << base);
00510 return ret;
00511 }
00512
00513 std::FILE *FMakeTemp(const StringPiece &base) {
00514 util::scoped_fd file(MakeTemp(base));
00515 return FDOpenOrThrow(file);
00516 }
00517
00518 int DupOrThrow(int fd) {
00519 int ret = dup(fd);
00520 UTIL_THROW_IF_ARG(ret == -1, FDException, (fd), "in duplicating the file descriptor");
00521 return ret;
00522 }
00523
00524 namespace {
00525
00526 bool TryName(int fd, std::string &out) {
00527 #if defined(_WIN32) || defined(_WIN64)
00528 return false;
00529 #else
00530 std::string name("/proc/self/fd/");
00531 std::ostringstream convert;
00532 convert << fd;
00533 name += convert.str();
00534
00535 struct stat sb;
00536 if (-1 == lstat(name.c_str(), &sb))
00537 return false;
00538 out.resize(sb.st_size + 1);
00539
00540 while (true) {
00541 ssize_t ret = readlink(name.c_str(), &out[0], out.size());
00542 if (-1 == ret)
00543 return false;
00544 if ((size_t)ret < out.size()) {
00545 out.resize(ret);
00546 break;
00547 }
00548
00549 out.resize(out.size() * 2);
00550 }
00551
00552 if (!out.empty() && out[0] != '/')
00553 return false;
00554 return true;
00555 #endif
00556 }
00557 }
00558
00559 std::string NameFromFD(int fd) {
00560 std::string ret;
00561 if (TryName(fd, ret)) return ret;
00562 switch (fd) {
00563 case 0: return "stdin";
00564 case 1: return "stdout";
00565 case 2: return "stderr";
00566 }
00567 ret = "fd ";
00568 std::ostringstream convert;
00569 convert << fd;
00570 ret += convert.str();
00571 return ret;
00572 }
00573
00574 }