large_file_test.cpp 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. /*
  2. * Distributed under the Boost Software License, Version 1.0.(See accompanying
  3. * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt.)
  4. *
  5. * See http://www.boost.org/libs/iostreams for documentation.
  6. *
  7. * Tests seeking with a file_descriptor using large file offsets.
  8. *
  9. * File: libs/iostreams/test/large_file_test.cpp
  10. * Date: Tue Dec 25 21:34:47 MST 2007
  11. * Copyright: 2007-2008 CodeRage, LLC
  12. * Author: Jonathan Turkanis
  13. * Contact: turkanis at coderage dot com
  14. */
  15. #include <cstdio> // SEEK_SET, etc.
  16. #include <ctime>
  17. #include <string>
  18. #include <boost/config.hpp> // BOOST_STRINGIZE
  19. #include <boost/detail/workaround.hpp>
  20. #include <boost/iostreams/detail/config/rtl.hpp>
  21. #include <boost/iostreams/detail/config/windows_posix.hpp>
  22. #include <boost/iostreams/detail/execute.hpp>
  23. #include <boost/iostreams/detail/ios.hpp>
  24. #include <boost/iostreams/device/file_descriptor.hpp>
  25. #include <boost/iostreams/device/mapped_file.hpp>
  26. #include <boost/iostreams/positioning.hpp>
  27. #include <boost/lexical_cast.hpp>
  28. #include <boost/test/test_tools.hpp>
  29. #include <boost/test/unit_test.hpp>
  30. #include <iostream>
  31. // OS-specific headers for low-level i/o.
  32. #include <fcntl.h> // file opening flags.
  33. #include <sys/stat.h> // file access permissions.
  34. #ifdef BOOST_IOSTREAMS_WINDOWS
  35. # include <io.h> // low-level file i/o.
  36. # define WINDOWS_LEAN_AND_MEAN
  37. # include <windows.h>
  38. # ifndef INVALID_SET_FILE_POINTER
  39. # define INVALID_SET_FILE_POINTER ((DWORD)-1)
  40. # endif
  41. #else
  42. # include <sys/types.h> // mode_t.
  43. # include <unistd.h> // low-level file i/o.
  44. #endif
  45. using namespace std;
  46. using namespace boost;
  47. using namespace boost::iostreams;
  48. using boost::unit_test::test_suite;
  49. //------------------Definition of constants-----------------------------------//
  50. const stream_offset gigabyte = 1073741824;
  51. const stream_offset file_size = // Some compilers complain about "8589934593"
  52. gigabyte * static_cast<stream_offset>(8) + static_cast<stream_offset>(1);
  53. const int offset_list[] =
  54. { 0, 1, 2, 3, 4, 5, 6, 7, 8, 7, 6, 5, 4, 3, 2, 1, // Seek by 1GB
  55. 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, // Seek by 2GB
  56. 6, 7, 5, 6, 4, 5, 3, 4, 2, 3, 1, 2,
  57. 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, // Seek by 3GB
  58. 5, 7, 4, 6, 3, 5, 2, 4, 1,
  59. 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, // Seek by 4GB
  60. 4, 7, 3, 6, 2, 5, 1, 4,
  61. 0, 5, 1, 6, 2, 7, 3, 8, 3, 7, 2, 6, 1, 5, // Seek by 5GB
  62. 0, 6, 1, 7, 2, 8, 2, 7, 1, 6, // Seek by 6GB
  63. 0, 7, 1, 8, 1, 7, // Seek by 7GB
  64. 0, 8, 0 }; // Seek by 8GB
  65. const int offset_list_length = sizeof(offset_list) / sizeof(int);
  66. #ifdef LARGE_FILE_TEMP
  67. # define BOOST_FILE_NAME BOOST_STRINGIZE(LARGE_FILE_TEMP)
  68. # define BOOST_KEEP_FILE false
  69. #else
  70. # define BOOST_FILE_NAME BOOST_STRINGIZE(LARGE_FILE_KEEP)
  71. # define BOOST_KEEP_FILE true
  72. #endif
  73. //------------------Definition of remove_large_file---------------------------//
  74. // Removes the large file
  75. void remove_large_file()
  76. {
  77. #ifdef BOOST_IOSTREAMS_WINDOWS
  78. DeleteFile(TEXT(BOOST_FILE_NAME));
  79. #else
  80. unlink(BOOST_FILE_NAME);
  81. #endif
  82. }
  83. //------------------Definition of large_file_exists---------------------------//
  84. // Returns true if the large file exists, has the correct size, and has been
  85. // modified since the last commit affecting this source file; if the file exists
  86. // but is invalid, deletes the file.
  87. bool large_file_exists()
  88. {
  89. // Last mod date
  90. time_t last_mod;
  91. #ifdef BOOST_IOSTREAMS_WINDOWS
  92. // Check existence
  93. WIN32_FIND_DATA info;
  94. HANDLE hnd = FindFirstFile(TEXT(BOOST_FILE_NAME), &info);
  95. if (hnd == INVALID_HANDLE_VALUE)
  96. return false;
  97. // Check size
  98. FindClose(hnd);
  99. stream_offset size =
  100. (static_cast<stream_offset>(info.nFileSizeHigh) << 32) +
  101. static_cast<stream_offset>(info.nFileSizeLow);
  102. if (size != file_size) {
  103. remove_large_file();
  104. return false;
  105. }
  106. // Fetch last mod date
  107. SYSTEMTIME stime;
  108. if (!FileTimeToSystemTime(&info.ftLastWriteTime, &stime)) {
  109. remove_large_file();
  110. return false;
  111. }
  112. tm ctime;
  113. ctime.tm_year = stime.wYear - 1900;
  114. ctime.tm_mon = stime.wMonth - 1;
  115. ctime.tm_mday = stime.wDay;
  116. ctime.tm_hour = stime.wHour;
  117. ctime.tm_min = stime.wMinute;
  118. ctime.tm_sec = stime.wSecond;
  119. ctime.tm_isdst = 0;
  120. last_mod = mktime(&ctime);
  121. #else
  122. // Check existence
  123. struct BOOST_IOSTREAMS_FD_STAT info;
  124. if (BOOST_IOSTREAMS_FD_STAT(BOOST_FILE_NAME, &info))
  125. return false;
  126. // Check size
  127. if (info.st_size != file_size) {
  128. remove_large_file();
  129. return false;
  130. }
  131. // Fetch last mod date
  132. last_mod = info.st_mtime;
  133. #endif
  134. // Fetch last mod date of this file ("large_file_test.cpp")
  135. string timestamp =
  136. "$Date$";
  137. if (timestamp.size() != 53) { // Length of auto-generated SVN timestamp
  138. remove_large_file();
  139. return false;
  140. }
  141. tm commit;
  142. try {
  143. commit.tm_year = lexical_cast<int>(timestamp.substr(7, 4)) - 1900;
  144. commit.tm_mon = lexical_cast<int>(timestamp.substr(12, 2)) - 1;
  145. commit.tm_mday = lexical_cast<int>(timestamp.substr(15, 2));
  146. commit.tm_hour = lexical_cast<int>(timestamp.substr(18, 2));
  147. commit.tm_min = lexical_cast<int>(timestamp.substr(21, 2));
  148. commit.tm_sec = lexical_cast<int>(timestamp.substr(24, 2));
  149. } catch (const bad_lexical_cast&) {
  150. remove_large_file();
  151. return false;
  152. }
  153. // If last commit was two days or more before file timestamp, existing
  154. // file is okay; otherwise, it must be regenerated (the two-day window
  155. // compensates for time zone differences)
  156. return difftime(last_mod, mktime(&commit)) >= 60 * 60 * 48;
  157. }
  158. //------------------Definition of map_large_file------------------------------//
  159. // Initializes the large file by mapping it in small segments. This is an
  160. // optimization for Win32; the straightforward implementation using WriteFile
  161. // and SetFilePointer (see the Borland workaround below) is painfully slow.
  162. bool map_large_file()
  163. {
  164. for (stream_offset z = 0; z <= 8; ++z) {
  165. try {
  166. mapped_file_params params;
  167. params.path = BOOST_FILE_NAME;
  168. params.offset = z * gigabyte;
  169. params.length = 1;
  170. params.mode = BOOST_IOS::out;
  171. mapped_file file(params);
  172. file.begin()[0] = static_cast<char>(z + 1);
  173. } catch (const std::exception&) {
  174. remove_large_file();
  175. return false;
  176. }
  177. }
  178. return true;
  179. }
  180. //------------------Definition of create_large_file---------------------------//
  181. // Creates and initializes the large file if it does not already exist. The file
  182. // looks like this:
  183. //
  184. // 0 1GB 2GB 3GB 4GB 5GB 6GB 7GB 8GB
  185. // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  186. // 1.....2.....3.....4.....5.....6.....7.....8.....9
  187. // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  188. //
  189. // where the characters 1-9 appear at offsets that are multiples of 1GB and the
  190. // dots represent uninitialized data.
  191. bool create_large_file()
  192. {
  193. // If file exists, has correct size, and is recent, we're done
  194. if (BOOST_KEEP_FILE && large_file_exists())
  195. return true;
  196. #ifdef BOOST_IOSTREAMS_WINDOWS
  197. // Create file
  198. HANDLE hnd =
  199. CreateFile(
  200. TEXT(BOOST_FILE_NAME),
  201. GENERIC_WRITE,
  202. 0,
  203. NULL,
  204. CREATE_ALWAYS,
  205. FILE_ATTRIBUTE_NORMAL,
  206. NULL
  207. );
  208. if (!hnd)
  209. return false;
  210. // Set file pointer
  211. LONG off_low = static_cast<LONG>(file_size & 0xffffffff);
  212. LONG off_high = static_cast<LONG>(file_size >> 32);
  213. if ( SetFilePointer(hnd, off_low, &off_high, FILE_BEGIN) ==
  214. INVALID_SET_FILE_POINTER &&
  215. GetLastError() != NO_ERROR )
  216. {
  217. CloseHandle(hnd);
  218. remove_large_file();
  219. return false;
  220. }
  221. // Set file size
  222. if (!SetEndOfFile(hnd)) {
  223. CloseHandle(hnd);
  224. remove_large_file();
  225. return false;
  226. }
  227. # if !defined(__BORLANDC__) || __BORLANDC__ < 0x582 || __BORLANDC__ >= 0x592
  228. // Close handle; all further access is via mapped_file
  229. CloseHandle(hnd);
  230. // Initialize file data
  231. return map_large_file();
  232. # else // Borland >= 5.8.2 and Borland < 5.9.2
  233. // Initialize file data (very slow, even though only 9 writes are required)
  234. for (stream_offset z = 0; z <= 8; ++z) {
  235. // Seek
  236. LONG off_low = static_cast<LONG>((z * gigabyte) & 0xffffffff); // == 0
  237. LONG off_high = static_cast<LONG>((z * gigabyte) >> 32);
  238. if ( SetFilePointer(hnd, off_low, &off_high, FILE_BEGIN) ==
  239. INVALID_SET_FILE_POINTER &&
  240. GetLastError() != NO_ERROR )
  241. {
  242. CloseHandle(hnd);
  243. remove_large_file();
  244. return false;
  245. }
  246. // Write a character
  247. char buf[1] = { z + 1 };
  248. DWORD result;
  249. BOOL success = WriteFile(hnd, buf, 1, &result, NULL);
  250. if (!success || result != 1) {
  251. CloseHandle(hnd);
  252. remove_large_file();
  253. return false;
  254. }
  255. }
  256. // Close file
  257. CloseHandle(hnd);
  258. return true;
  259. # endif // Borland workaround
  260. #else // #ifdef BOOST_IOSTREAMS_WINDOWS
  261. // Create file
  262. int oflag = O_WRONLY | O_CREAT;
  263. #ifdef _LARGEFILE64_SOURCE
  264. oflag |= O_LARGEFILE;
  265. #endif
  266. mode_t pmode =
  267. S_IRUSR | S_IWUSR |
  268. S_IRGRP | S_IWGRP |
  269. S_IROTH | S_IWOTH;
  270. int fd = BOOST_IOSTREAMS_FD_OPEN(BOOST_FILE_NAME, oflag, pmode);
  271. if (fd == -1)
  272. return false;
  273. // Set file size
  274. if (BOOST_IOSTREAMS_FD_TRUNCATE(fd, file_size)) {
  275. BOOST_IOSTREAMS_FD_CLOSE(fd);
  276. return false;
  277. }
  278. # ifndef __CYGWIN__
  279. // Initialize file data
  280. for (int z = 0; z <= 8; ++z) {
  281. // Seek
  282. BOOST_IOSTREAMS_FD_OFFSET off =
  283. BOOST_IOSTREAMS_FD_SEEK(
  284. fd,
  285. static_cast<BOOST_IOSTREAMS_FD_OFFSET>(z * gigabyte),
  286. SEEK_SET
  287. );
  288. if (off == -1) {
  289. BOOST_IOSTREAMS_FD_CLOSE(fd);
  290. return false;
  291. }
  292. // Write a character
  293. char buf[1] = { z + 1 };
  294. if (BOOST_IOSTREAMS_FD_WRITE(fd, buf, 1) == -1) {
  295. BOOST_IOSTREAMS_FD_CLOSE(fd);
  296. return false;
  297. }
  298. }
  299. // Close file
  300. BOOST_IOSTREAMS_FD_CLOSE(fd);
  301. return true;
  302. # else // Cygwin
  303. // Close descriptor; all further access is via mapped_file
  304. BOOST_IOSTREAMS_FD_CLOSE(fd);
  305. // Initialize file data
  306. return map_large_file();
  307. # endif
  308. #endif // #ifdef BOOST_IOSTREAMS_WINDOWS
  309. }
  310. //------------------Definition of large_file----------------------------------//
  311. // RAII utility
  312. class large_file {
  313. public:
  314. large_file() { exists_ = create_large_file(); }
  315. ~large_file() { if (!BOOST_KEEP_FILE) remove_large_file(); }
  316. bool exists() const { return exists_; }
  317. const char* path() const { return BOOST_FILE_NAME; }
  318. private:
  319. bool exists_;
  320. };
  321. //------------------Definition of check_character-----------------------------//
  322. // Verify that the given file contains the given character at the current
  323. // position
  324. bool check_character(file_descriptor_source& file, char value)
  325. {
  326. char buf[1];
  327. std::streamsize amt;
  328. BOOST_CHECK_NO_THROW(amt = file.read(buf, 1));
  329. BOOST_CHECK_MESSAGE(amt == 1, "failed reading character");
  330. BOOST_CHECK_NO_THROW(file.seek(-1, BOOST_IOS::cur));
  331. return buf[0] == value;
  332. }
  333. //------------------Definition of large_file_test-----------------------------//
  334. void large_file_test()
  335. {
  336. BOOST_REQUIRE_MESSAGE(
  337. sizeof(stream_offset) >= 8,
  338. "large offsets not supported"
  339. );
  340. // Prepare file and file descriptor
  341. large_file large;
  342. file_descriptor_source file;
  343. BOOST_REQUIRE_MESSAGE(
  344. large.exists(), "failed creating file \"" << BOOST_FILE_NAME << '"'
  345. );
  346. BOOST_CHECK_NO_THROW(file.open(large.path(), BOOST_IOS::binary));
  347. // Test seeking using ios_base::beg
  348. for (int z = 0; z < offset_list_length; ++z) {
  349. char value = offset_list[z] + 1;
  350. stream_offset off =
  351. static_cast<stream_offset>(offset_list[z]) * gigabyte;
  352. BOOST_CHECK_NO_THROW(file.seek(off, BOOST_IOS::beg));
  353. BOOST_CHECK_MESSAGE(
  354. check_character(file, value),
  355. "failed validating seek"
  356. );
  357. }
  358. // Test seeking using ios_base::end
  359. for (int z = 0; z < offset_list_length; ++z) {
  360. char value = offset_list[z] + 1;
  361. stream_offset off =
  362. -static_cast<stream_offset>(8 - offset_list[z]) * gigabyte - 1;
  363. BOOST_CHECK_NO_THROW(file.seek(off, BOOST_IOS::end));
  364. BOOST_CHECK_MESSAGE(
  365. check_character(file, value),
  366. "failed validating seek"
  367. );
  368. }
  369. // Test seeking using ios_base::cur
  370. for (int next, cur = 0, z = 0; z < offset_list_length; ++z, cur = next) {
  371. next = offset_list[z];
  372. char value = offset_list[z] + 1;
  373. stream_offset off = static_cast<stream_offset>(next - cur) * gigabyte;
  374. BOOST_CHECK_NO_THROW(file.seek(off, BOOST_IOS::cur));
  375. BOOST_CHECK_MESSAGE(
  376. check_character(file, value),
  377. "failed validating seek"
  378. );
  379. }
  380. }
  381. test_suite* init_unit_test_suite(int, char* [])
  382. {
  383. test_suite* test = BOOST_TEST_SUITE("execute test");
  384. test->add(BOOST_TEST_CASE(&large_file_test));
  385. return test;
  386. }