1 /**************************************************************************
3 * Copyright 2013-2014 RAD Game Tools and Valve Software
4 * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 **************************************************************************/
27 // File: vogl_miniz_zip_test.cpp
28 #include "vogl_miniz_zip.h"
29 #include "vogl_core.h"
30 #include "vogl_cfile_stream.h"
31 #include "vogl_rand.h"
32 #include "vogl_unique_ptr.h"
33 #include "vogl_file_utils.h"
34 #include "vogl_find_files.h"
38 static uint8 *read_rand_data(data_stream &rand_stream, uint64_t file_size)
40 const uint64_t rand_stream_size = rand_stream.get_size();
42 if (file_size > rand_stream_size)
45 uint64_t max_ofs = rand_stream.get_size() - file_size;
46 uint64_t src_file_ofs = g_thread_safe_random.urand64_inclusive(0, max_ofs);
48 if (!rand_stream.seek(src_file_ofs, false))
51 if (file_size != static_cast<size_t>(file_size))
54 uint8 *p = static_cast<uint8 *>(vogl_malloc(static_cast<size_t>(file_size)));
58 if (rand_stream.read64(p, file_size) != file_size)
69 dynamic_string m_name;
73 zip_file_desc(const dynamic_string &name, uint64_t size, uint32 crc)
74 : m_name(name), m_size(size), m_crc(crc)
78 typedef vogl::vector<zip_file_desc> zip_file_desc_vec;
80 typedef vogl::vector<uint64_t> uint64_vec;
82 static void create_rand_file_sizes(uint64_vec &file_sizes, uint num_files, uint64_t min_file_size = 0, uint64_t max_file_size = cUINT64_MAX)
84 file_sizes.resize(num_files);
85 for (uint i = 0; i < num_files; i++)
86 file_sizes[i] = g_thread_safe_random.urand64_inclusive(min_file_size, max_file_size);
92 printf("Failure on line %u\n", __LINE__); \
97 static bool mz_zip_create_random_file_archive(data_stream &rand_stream, const char *pZip_filename, bool zip64, uint64_t min_file_size, uint64_t max_file_size, uint min_files, uint max_files, zip_file_desc_vec &files, mz_zip_archive *pSrc_zip)
99 const uint total_files = g_thread_safe_random.urand_inclusive(min_files, max_files);
101 uint64_vec file_sizes;
102 create_rand_file_sizes(file_sizes, total_files, min_file_size, math::minimum<uint64_t>(rand_stream.get_size(), max_file_size));
105 mz_zip_zero_struct(&zip);
107 uint proceeding_size = g_thread_safe_random.urand(0, 1000000);
109 printf("Creating archive %s, total files %u\n", pZip_filename, total_files);
111 if (!mz_zip_writer_init_file(&zip, pZip_filename, proceeding_size, zip64 ? MZ_ZIP_FLAG_WRITE_ZIP64 : 0))
114 dynamic_string_array filenames;
115 for (uint i = 0; i < total_files; i++)
116 filenames.push_back(dynamic_string(cVarArg, "xxxxfile_%u", i));
118 switch (g_thread_safe_random.irand_inclusive(0, 2))
125 filenames.shuffle(g_thread_safe_random);
129 dynamic_string temp_filename(cVarArg, "temp_%u.bin", g_thread_safe_random.urand32());
131 for (uint i = 0; i < total_files; i++)
133 const dynamic_string &name = filenames[i];
135 //uint level = g_thread_safe_random.irand_inclusive(0, 10);
136 uint level = g_thread_safe_random.irand_inclusive(0, 1);
141 uint64_t actual_size = file_sizes[i];
143 uint8 *p = read_rand_data(rand_stream, actual_size);
147 if (actual_size != static_cast<size_t>(actual_size))
155 if ((zip.m_archive_size + (actual_size * 101ULL) / 100 + mz_zip_get_central_dir_size(&zip) + 4096) > 0xFFFF0000)
159 bool add_from_mem = g_thread_safe_random.get_bit();
161 printf("Adding file %s, size %s, add from mem: %u\n", name.get_ptr(), uint64_to_string_with_commas(actual_size).get_ptr(), add_from_mem);
163 files.push_back(zip_file_desc(name, actual_size, (uint32)mz_crc32(MZ_CRC32_INIT, p, static_cast<size_t>(actual_size))));
165 dynamic_string comment_str;
166 comment_str.reserve(512);
167 uint l = g_thread_safe_random.urand_inclusive(0, 300);
169 for (uint i = 0; i < l; i++)
170 comment_str.append_char((char)g_thread_safe_random.urand_inclusive(32, 127));
174 if (!mz_zip_writer_add_mem_ex(&zip, name.get_ptr(), p, static_cast<size_t>(actual_size), comment_str.get_ptr(), comment_str.get_len(), level, 0, 0))
182 if (!file_utils::write_buf_to_file(temp_filename.get_ptr(), p, static_cast<size_t>(actual_size)))
188 if (!mz_zip_writer_add_file(&zip, name.get_ptr(), temp_filename.get_ptr(), comment_str.get_ptr(), comment_str.get_len(), level))
200 uint n = g_thread_safe_random.urand_inclusive(0, VOGL_MIN(100, pSrc_zip->m_total_files));
202 printf("Copying %u files from source archive\n", n);
204 vogl::vector<uint> used_files;
206 for (uint i = 0; i < n; i++)
211 src_file_index = g_thread_safe_random.urand(0, pSrc_zip->m_total_files);
212 if (used_files.find(src_file_index) < 0)
215 used_files.push_back(src_file_index);
217 mz_zip_archive_file_stat stat;
218 if (!mz_zip_file_stat(pSrc_zip, src_file_index, &stat))
223 if (zip.m_total_files == 0xFFFF)
226 if ((zip.m_archive_size + stat.m_comp_size + mz_zip_get_central_dir_size(&zip) + 4096) > 0xFFFF0000)
231 if (zip.m_total_files == 0xFFFFFFFF)
235 if (!mz_zip_writer_add_from_other_zip(&zip, pSrc_zip, src_file_index))
240 if (!mz_zip_writer_finalize_archive(&zip))
242 mz_zip_writer_end(&zip);
246 printf("Finished archive %s, total files %u, total size %s\n", pZip_filename, total_files, uint64_to_string_with_commas(zip.m_archive_size).get_ptr());
248 if (!mz_zip_writer_end(&zip))
258 printf("Failed on line %u\n", __LINE__); \
262 static bool shuffle_order_of_archive(const char *pSrc_zip, const char *pDst_zip)
264 mz_zip_archive src_zip;
265 mz_zip_zero_struct(&src_zip);
267 if (!mz_zip_reader_init_file(&src_zip, pSrc_zip, 0, 0, 0))
270 mz_zip_archive dst_zip;
271 mz_zip_zero_struct(&dst_zip);
273 if (!mz_zip_writer_init_file(&dst_zip, pDst_zip, 0, MZ_ZIP_FLAG_WRITE_ALLOW_READING | (mz_zip_is_zip64(&src_zip) ? MZ_ZIP_FLAG_WRITE_ZIP64 : 0)))
275 mz_zip_reader_end(&src_zip);
279 vogl::vector<uint> file_order(src_zip.m_total_files);
280 for (uint i = 0; i < src_zip.m_total_files; i++)
282 file_order.shuffle(g_thread_safe_random);
284 for (uint i = 0; i < src_zip.m_total_files; i++)
286 printf("Copying file %u, %3.2f%%\n", file_order[i], (i * 100.0f) / src_zip.m_total_files);
287 if (!mz_zip_writer_add_from_other_zip(&dst_zip, &src_zip, file_order[i]))
289 mz_zip_reader_end(&src_zip);
290 mz_zip_writer_end(&dst_zip);
294 mz_zip_archive_file_stat stat;
295 mz_bool status = mz_zip_file_stat(&dst_zip, i, &stat);
299 status = mz_zip_locate_file(&dst_zip, stat.m_filename, NULL, 0, &index);
303 dynamic_string_array filenames;
304 for (uint j = 0; j < dst_zip.m_total_files; j++)
306 mz_zip_archive_file_stat file_stat;
307 mz_zip_file_stat(&dst_zip, j, &file_stat);
308 filenames.push_back(dynamic_string(file_stat.m_filename));
311 uint prev_num = filenames.size();
313 uint cur_num = filenames.size();
315 CHECK(prev_num != cur_num);
318 CHECK(mz_zip_validate_file(&dst_zip, i, 0));
321 mz_zip_reader_end(&src_zip);
323 if (!mz_zip_writer_finalize_archive(&dst_zip))
325 mz_zip_writer_end(&dst_zip);
329 if (!mz_zip_writer_end(&dst_zip))
335 bool validate_archives_in_dir(const char *pDir)
338 if (!finder.find(pDir, find_files::cFlagRecursive | find_files::cFlagAllowFiles))
341 uint num_failures = 0;
344 dynamic_string_array failures;
346 dynamic_string temp_filename(vogl::file_utils::generate_temp_filename("/tmp/"));
348 for (uint file_index = 0; file_index < finder.get_files().size(); file_index++)
350 dynamic_string filename(finder.get_files()[file_index].m_fullname);
352 printf("Validating archive: %s\n", filename.get_ptr());
354 mz_zip_error last_zip_error = MZ_ZIP_NO_ERROR;
355 if (!mz_zip_validate_file_archive(filename.get_ptr(), 0, &last_zip_error))
357 if ((last_zip_error == MZ_ZIP_UNSUPPORTED_METHOD) || (last_zip_error == MZ_ZIP_UNSUPPORTED_ENCRYPTION))
359 printf("Failed, unsupported method or archive is encrypted\n");
363 printf("FAILED - trying to 7z\n");
365 int result = system(dynamic_string(cVarArg, "7z -p t \"%s\"", filename.get_ptr()).get_ptr());
368 printf("7z succeeded but miniz_zip failed, so this is a FAILURE!\n");
370 failures.push_back(filename);
374 printf("7z also failed, so all is ok\n");
382 printf("Validating with 7z:\n");
383 int result = system(dynamic_string(cVarArg, "7z t \"%s\"", filename.get_ptr()).get_ptr());
385 printf("Couldn't validate original archive with 7z!\n");
388 printf("Shuffling archive %s:\n", temp_filename.get_ptr());
389 if (!shuffle_order_of_archive(filename.get_ptr(), temp_filename.get_ptr()))
392 printf("Validating with miniz_zip:\n");
393 if (!mz_zip_validate_file_archive(temp_filename.get_ptr(), 0, &last_zip_error))
396 printf("Validating with 7z:\n");
397 int result = system(dynamic_string(cVarArg, "7z t \"%s\"", temp_filename.get_ptr()).get_ptr());
406 printf("Total failures: %u, total good: %u\n", num_failures, num_good);
407 for (uint i = 0; i < failures.size(); i++)
408 printf("%s\n", failures[i].get_ptr());
414 g_thread_safe_random.seed(1000);
416 //CHECK(mz_zip_validate_file_archive("/home/richg/temp/blah.zip", 0, NULL));
417 //CHECK(mz_zip_validate_file_archive("big.zip", 0, NULL));
418 //CHECK(shuffle_order_of_archive("/home/richg/temp/blah.zip", "big.zip"));
420 //CHECK(shuffle_order_of_archive("big.zip", "big2.zip"));
421 //CHECK(mz_zip_validate_file_archive("big2.zip", 0, NULL));
425 CHECK(validate_archives_in_dir("/media/AB80-6D4D/*.zip"));
428 //CHECK(validate_archives_in_dir("/home/richg/dev/raddebugger/bin/temp4/*.zip"));
431 //CHECK(validate_archives_in_dir("/media/Green/dev/*.zip"));
433 //CHECK(validate_archives_in_dir("/home/richg/temp/*.zip"));
437 //CHECK(mz_zip_validate_file_archive("g1.zip", 0));
441 //void *p = calloc(0xFFFFFFFF, 1);
442 //bool status = mz_zip_add_mem_to_archive_file_in_place("x2.zip", "1.bin", p, 0xFFFFFFFF, NULL, 0, 1, false);
445 CHECK(mz_zip_validate_file_archive("x2.zip", MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG));
447 CHECK(mz_zip_validate_file_archive("x3.zip", MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG));
449 CHECK(mz_zip_validate_file_archive("x.zip", MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG));
451 mz_zip_archive zip_reader;
452 mz_zip_zero_struct(&zip_reader);
453 CHECK(mz_zip_reader_init_file(&zip_reader, "x2.zip", 0));
456 mz_zip_zero_struct(&zip);
458 CHECK(mz_zip_writer_init_file(&zip, "x3.zip", 0, MZ_ZIP_FLAG_WRITE_ZIP64));
459 CHECK(mz_zip_writer_add_from_zip_reader(&zip, &zip_reader, 0));
460 CHECK(mz_zip_writer_finalize_archive(&zip));
461 CHECK(mz_zip_writer_end(&zip));
465 //bool success = validate_archive("z2.zip", true);
471 rm.seed_from_urandom();
473 mz_zip_archive src_zip64;
474 mz_zip_zero_struct(&src_zip64);
475 CHECK(mz_zip_reader_init_file(&src_zip64, "/home/richg/temp/a.zip", 0, 0, 0));
476 //CHECK(mz_zip_is_zip64(&src_zip64));
478 mz_zip_archive src_zip32;
479 mz_zip_zero_struct(&src_zip32);
480 CHECK(mz_zip_reader_init_file(&src_zip32, "/home/richg/temp/a.zip", 0, 0, 0));
481 //CHECK(!mz_zip_is_zip64(&src_zip32));
483 cfile_stream rand_stream("/home/richg/temp/output1.bin", cDataStreamReadable | cDataStreamSeekable);
484 if (!rand_stream.is_opened())
487 int zip_base = rm.urand_inclusive('a', 'z');
488 int start_seed = g_thread_safe_random.urand(1, 10000);
490 dynamic_string temp_filename(cVarArg, "temp_%u.bin", g_thread_safe_random.urand32());
492 for (uint t = 0; t < 10000; t++)
494 uint32 seed = t + start_seed;
495 printf("******* Pass %u, seed %u\n", t, seed);
497 g_thread_safe_random.seed(seed);
499 dynamic_string zip_filename(cVarArg, "%c%u.zip", zip_base, t + 1);
501 dynamic_string full_zip_filename(zip_filename);
502 file_utils::full_path(full_zip_filename);
504 bool zip64 = g_thread_safe_random.get_bit();
506 bool test_big_files = g_thread_safe_random.get_bit();
510 max_files = (uint)g_thread_safe_random.urand64_inclusive(0, 10);
512 max_files = (uint)g_thread_safe_random.urand64_inclusive(0, zip64 ? 200000 : 65535);
514 printf("Max files: %u, zip64: %u\n", max_files, zip64);
518 uint64_t min_file_size = 0;
519 uint64_t max_file_size = 1000000; //zip64 ? 0xFFFFFFFFFF : 0xFFFFFFFF;
522 max_file_size = 1024;
525 zip_file_desc_vec file_descs;
526 success = mz_zip_create_random_file_archive(rand_stream, zip_filename.get_ptr(), zip64, min_file_size, max_file_size, 0, max_files, file_descs, zip64 ? (g_thread_safe_random.get_bit() ? &src_zip64 : &src_zip32) : &src_zip32);
529 uint num_actual_files = file_descs.size();
531 printf("******* Validating archive %s using miniz_zip\n", full_zip_filename.get_ptr());
533 mz_zip_error last_zip_error = MZ_ZIP_NO_ERROR;
534 success = mz_zip_validate_file_archive(zip_filename.get_ptr(), ((g_thread_safe_random.get_bit() && (file_descs.size() < 30000)) ? MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY : 0) | MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG, &last_zip_error);
537 printf("******* Validating archive %s using unzip\n", full_zip_filename.get_ptr());
539 int result = system(dynamic_string(cVarArg, "unzip -t \"%s\"", full_zip_filename.get_ptr()).get_ptr());
540 if ((num_actual_files == 0) && (result == 256))
548 printf("******* Validating archive %s using 7z\n", full_zip_filename.get_ptr());
550 result = system(dynamic_string(cVarArg, "7z t \"%s\"", full_zip_filename.get_ptr()).get_ptr());
553 printf("Shuffling archive %s:\n", full_zip_filename.get_ptr());
554 if (!shuffle_order_of_archive(full_zip_filename.get_ptr(), temp_filename.get_ptr()))
557 printf("Validating with miniz_zip:\n");
558 if (!mz_zip_validate_file_archive(temp_filename.get_ptr(), 0, &last_zip_error))
561 printf("Validating with 7z:\n");
562 result = system(dynamic_string(cVarArg, "7z t \"%s\"", temp_filename.get_ptr()).get_ptr());
567 // this func potentially leaks if something fails
568 mz_zip_reader_end(&src_zip32);
569 mz_zip_reader_end(&src_zip64);