git.cworth.org Git - vogl/blob - src/voglcore/vogl_miniz_zip_test.cpp

   1 /**************************************************************************
   2  *
   3  * Copyright 2013-2014 RAD Game Tools and Valve Software
   4  * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a copy
   8  * of this software and associated documentation files (the "Software"), to deal
   9  * in the Software without restriction, including without limitation the rights
  10  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11  * copies of the Software, and to permit persons to whom the Software is
  12  * furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice shall be included in
  15  * all copies or substantial portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23  * THE SOFTWARE.
  24  *
  25  **************************************************************************/
  26
  27 // File: vogl_miniz_zip_test.cpp
  28 #include "vogl_miniz_zip.h"
  29 #include "vogl_core.h"
  30 #include "vogl_cfile_stream.h"
  31 #include "vogl_rand.h"
  32 #include "vogl_unique_ptr.h"
  33 #include "vogl_file_utils.h"
  34 #include "vogl_find_files.h"
  35
  36 namespace vogl
  37 {
  38     static uint8 *read_rand_data(data_stream &rand_stream, uint64_t file_size)
  39     {
  40         const uint64_t rand_stream_size = rand_stream.get_size();
  41
  42         if (file_size > rand_stream_size)
  43             return NULL;
  44
  45         uint64_t max_ofs = rand_stream.get_size() - file_size;
  46         uint64_t src_file_ofs = g_thread_safe_random.urand64_inclusive(0, max_ofs);
  47
  48         if (!rand_stream.seek(src_file_ofs, false))
  49             return NULL;
  50
  51         if (file_size != static_cast<size_t>(file_size))
  52             return NULL;
  53
  54         uint8 *p = static_cast<uint8 *>(vogl_malloc(static_cast<size_t>(file_size)));
  55         if (!p)
  56             return NULL;
  57
  58         if (rand_stream.read64(p, file_size) != file_size)
  59         {
  60             vogl_free(p);
  61             return NULL;
  62         }
  63
  64         return p;
  65     }
  66
  67     struct zip_file_desc
  68     {
  69         dynamic_string m_name;
  70         uint64_t m_size;
  71         uint32 m_crc;
  72
  73         zip_file_desc(const dynamic_string &name, uint64_t size, uint32 crc)
  74             : m_name(name), m_size(size), m_crc(crc)
  75         {
  76         }
  77     };
  78     typedef vogl::vector<zip_file_desc> zip_file_desc_vec;
  79
  80     typedef vogl::vector<uint64_t> uint64_vec;
  81
  82     static void create_rand_file_sizes(uint64_vec &file_sizes, uint num_files, uint64_t min_file_size = 0, uint64_t max_file_size = cUINT64_MAX)
  83     {
  84         file_sizes.resize(num_files);
  85         for (uint i = 0; i < num_files; i++)
  86             file_sizes[i] = g_thread_safe_random.urand64_inclusive(min_file_size, max_file_size);
  87     }
  88
  89 #define FAIL                                      \
  90     do                                            \
  91     {                                             \
  92         printf("Failure on line %u\n", __LINE__); \
  93         VOGL_ASSERT(0);                         \
  94         return false;                             \
  95     } while (0)
  96
  97     static bool mz_zip_create_random_file_archive(data_stream &rand_stream, const char *pZip_filename, bool zip64, uint64_t min_file_size, uint64_t max_file_size, uint min_files, uint max_files, zip_file_desc_vec &files, mz_zip_archive *pSrc_zip)
  98     {
  99         const uint total_files = g_thread_safe_random.urand_inclusive(min_files, max_files);
 100
 101         uint64_vec file_sizes;
 102         create_rand_file_sizes(file_sizes, total_files, min_file_size, math::minimum<uint64_t>(rand_stream.get_size(), max_file_size));
 103
 104         mz_zip_archive zip;
 105         mz_zip_zero_struct(&zip);
 106
 107         uint proceeding_size = g_thread_safe_random.urand(0, 1000000);
 108
 109         printf("Creating archive %s, total files %u\n", pZip_filename, total_files);
 110
 111         if (!mz_zip_writer_init_file(&zip, pZip_filename, proceeding_size, zip64 ? MZ_ZIP_FLAG_WRITE_ZIP64 : 0))
 112             FAIL;
 113
 114         dynamic_string_array filenames;
 115         for (uint i = 0; i < total_files; i++)
 116             filenames.push_back(dynamic_string(cVarArg, "xxxxfile_%u", i));
 117
 118         switch (g_thread_safe_random.irand_inclusive(0, 2))
 119         {
 120             case 0:
 121                 break;
 122             case 1:
 123                 filenames.reverse();
 124             case 2:
 125                 filenames.shuffle(g_thread_safe_random);
 126                 break;
 127         }
 128
 129         dynamic_string temp_filename(cVarArg, "temp_%u.bin", g_thread_safe_random.urand32());
 130
 131         for (uint i = 0; i < total_files; i++)
 132         {
 133             const dynamic_string &name = filenames[i];
 134
 135             //uint level = g_thread_safe_random.irand_inclusive(0, 10);
 136             uint level = g_thread_safe_random.irand_inclusive(0, 1);
 137
 138             // HACK HACK
 139             //level = 0;
 140
 141             uint64_t actual_size = file_sizes[i];
 142
 143             uint8 *p = read_rand_data(rand_stream, actual_size);
 144             if (!p)
 145                 FAIL;
 146
 147             if (actual_size != static_cast<size_t>(actual_size))
 148             {
 149                 vogl_free(p);
 150                 FAIL;
 151             }
 152
 153             if (!zip64)
 154             {
 155                 if ((zip.m_archive_size + (actual_size * 101ULL) / 100 + mz_zip_get_central_dir_size(&zip) + 4096) > 0xFFFF0000)
 156                     break;
 157             }
 158
 159             bool add_from_mem = g_thread_safe_random.get_bit();
 160
 161             printf("Adding file %s, size %s, add from mem: %u\n", name.get_ptr(), uint64_to_string_with_commas(actual_size).get_ptr(), add_from_mem);
 162
 163             files.push_back(zip_file_desc(name, actual_size, (uint32)mz_crc32(MZ_CRC32_INIT, p, static_cast<size_t>(actual_size))));
 164
 165             dynamic_string comment_str;
 166             comment_str.reserve(512);
 167             uint l = g_thread_safe_random.urand_inclusive(0, 300);
 168
 169             for (uint i = 0; i < l; i++)
 170                 comment_str.append_char((char)g_thread_safe_random.urand_inclusive(32, 127));
 171
 172             if (add_from_mem)
 173             {
 174                 if (!mz_zip_writer_add_mem_ex(&zip, name.get_ptr(), p, static_cast<size_t>(actual_size), comment_str.get_ptr(), comment_str.get_len(), level, 0, 0))
 175                 {
 176                     vogl_free(p);
 177                     FAIL;
 178                 }
 179             }
 180             else
 181             {
 182                 if (!file_utils::write_buf_to_file(temp_filename.get_ptr(), p, static_cast<size_t>(actual_size)))
 183                 {
 184                     vogl_free(p);
 185                     FAIL;
 186                 }
 187
 188                 if (!mz_zip_writer_add_file(&zip, name.get_ptr(), temp_filename.get_ptr(), comment_str.get_ptr(), comment_str.get_len(), level))
 189                 {
 190                     vogl_free(p);
 191                     FAIL;
 192                 }
 193             }
 194
 195             vogl_free(p);
 196         }
 197
 198         if (pSrc_zip)
 199         {
 200             uint n = g_thread_safe_random.urand_inclusive(0, VOGL_MIN(100, pSrc_zip->m_total_files));
 201
 202             printf("Copying %u files from source archive\n", n);
 203
 204             vogl::vector<uint> used_files;
 205
 206             for (uint i = 0; i < n; i++)
 207             {
 208                 uint src_file_index;
 209                 for (;;)
 210                 {
 211                     src_file_index = g_thread_safe_random.urand(0, pSrc_zip->m_total_files);
 212                     if (used_files.find(src_file_index) < 0)
 213                         break;
 214                 }
 215                 used_files.push_back(src_file_index);
 216
 217                 mz_zip_archive_file_stat stat;
 218                 if (!mz_zip_file_stat(pSrc_zip, src_file_index, &stat))
 219                     FAIL;
 220
 221                 if (!zip64)
 222                 {
 223                     if (zip.m_total_files == 0xFFFF)
 224                         break;
 225
 226                     if ((zip.m_archive_size + stat.m_comp_size + mz_zip_get_central_dir_size(&zip) + 4096) > 0xFFFF0000)
 227                         break;
 228                 }
 229                 else
 230                 {
 231                     if (zip.m_total_files == 0xFFFFFFFF)
 232                         break;
 233                 }
 234
 235                 if (!mz_zip_writer_add_from_other_zip(&zip, pSrc_zip, src_file_index))
 236                     FAIL;
 237             }
 238         }
 239
 240         if (!mz_zip_writer_finalize_archive(&zip))
 241         {
 242             mz_zip_writer_end(&zip);
 243             FAIL;
 244         }
 245
 246         printf("Finished archive %s, total files %u, total size %s\n", pZip_filename, total_files, uint64_to_string_with_commas(zip.m_archive_size).get_ptr());
 247
 248         if (!mz_zip_writer_end(&zip))
 249             FAIL;
 250
 251         return true;
 252     }
 253
 254 #define CHECK(x)                                 \
 255     if (!(x))                                    \
 256     {                                            \
 257         VOGL_ASSERT(0);                        \
 258         printf("Failed on line %u\n", __LINE__); \
 259         return false;                            \
 260     }
 261
 262     static bool shuffle_order_of_archive(const char *pSrc_zip, const char *pDst_zip)
 263     {
 264         mz_zip_archive src_zip;
 265         mz_zip_zero_struct(&src_zip);
 266
 267         if (!mz_zip_reader_init_file(&src_zip, pSrc_zip, 0, 0, 0))
 268             FAIL;
 269
 270         mz_zip_archive dst_zip;
 271         mz_zip_zero_struct(&dst_zip);
 272
 273         if (!mz_zip_writer_init_file(&dst_zip, pDst_zip, 0, MZ_ZIP_FLAG_WRITE_ALLOW_READING | (mz_zip_is_zip64(&src_zip) ? MZ_ZIP_FLAG_WRITE_ZIP64 : 0)))
 274         {
 275             mz_zip_reader_end(&src_zip);
 276             FAIL;
 277         }
 278
 279         vogl::vector<uint> file_order(src_zip.m_total_files);
 280         for (uint i = 0; i < src_zip.m_total_files; i++)
 281             file_order[i] = i;
 282         file_order.shuffle(g_thread_safe_random);
 283
 284         for (uint i = 0; i < src_zip.m_total_files; i++)
 285         {
 286             printf("Copying file %u, %3.2f%%\n", file_order[i], (i * 100.0f) / src_zip.m_total_files);
 287             if (!mz_zip_writer_add_from_other_zip(&dst_zip, &src_zip, file_order[i]))
 288             {
 289                 mz_zip_reader_end(&src_zip);
 290                 mz_zip_writer_end(&dst_zip);
 291                 FAIL;
 292             }
 293
 294             mz_zip_archive_file_stat stat;
 295             mz_bool status = mz_zip_file_stat(&dst_zip, i, &stat);
 296             CHECK(status);
 297
 298             mz_uint32 index = 0;
 299             status = mz_zip_locate_file(&dst_zip, stat.m_filename, NULL, 0, &index);
 300             CHECK(status);
 301             if (index != i)
 302             {
 303                 dynamic_string_array filenames;
 304                 for (uint j = 0; j < dst_zip.m_total_files; j++)
 305                 {
 306                     mz_zip_archive_file_stat file_stat;
 307                     mz_zip_file_stat(&dst_zip, j, &file_stat);
 308                     filenames.push_back(dynamic_string(file_stat.m_filename));
 309                 }
 310                 filenames.sort();
 311                 uint prev_num = filenames.size();
 312                 filenames.unique();
 313                 uint cur_num = filenames.size();
 314
 315                 CHECK(prev_num != cur_num);
 316             }
 317
 318             CHECK(mz_zip_validate_file(&dst_zip, i, 0));
 319         }
 320
 321         mz_zip_reader_end(&src_zip);
 322
 323         if (!mz_zip_writer_finalize_archive(&dst_zip))
 324         {
 325             mz_zip_writer_end(&dst_zip);
 326             FAIL;
 327         }
 328
 329         if (!mz_zip_writer_end(&dst_zip))
 330             FAIL;
 331
 332         return true;
 333     }
 334
 335     bool validate_archives_in_dir(const char *pDir)
 336     {
 337         find_files finder;
 338         if (!finder.find(pDir, find_files::cFlagRecursive | find_files::cFlagAllowFiles))
 339             return true;
 340
 341         uint num_failures = 0;
 342         uint num_good = 0;
 343
 344         dynamic_string_array failures;
 345
 346         dynamic_string temp_filename(vogl::file_utils::generate_temp_filename("/tmp/"));
 347
 348         for (uint file_index = 0; file_index < finder.get_files().size(); file_index++)
 349         {
 350             dynamic_string filename(finder.get_files()[file_index].m_fullname);
 351
 352             printf("Validating archive: %s\n", filename.get_ptr());
 353
 354             mz_zip_error last_zip_error = MZ_ZIP_NO_ERROR;
 355             if (!mz_zip_validate_file_archive(filename.get_ptr(), 0, &last_zip_error))
 356             {
 357                 if ((last_zip_error == MZ_ZIP_UNSUPPORTED_METHOD) || (last_zip_error == MZ_ZIP_UNSUPPORTED_ENCRYPTION))
 358                 {
 359                     printf("Failed, unsupported method or archive is encrypted\n");
 360                 }
 361                 else
 362                 {
 363                     printf("FAILED - trying to 7z\n");
 364
 365                     int result = system(dynamic_string(cVarArg, "7z -p t \"%s\"", filename.get_ptr()).get_ptr());
 366                     if (result == 0)
 367                     {
 368                         printf("7z succeeded but miniz_zip failed, so this is a FAILURE!\n");
 369                         num_failures++;
 370                         failures.push_back(filename);
 371                     }
 372                     else
 373                     {
 374                         printf("7z also failed, so all is ok\n");
 375                     }
 376                 }
 377             }
 378             else
 379             {
 380                 num_good++;
 381
 382                 printf("Validating with 7z:\n");
 383                 int result = system(dynamic_string(cVarArg, "7z t \"%s\"", filename.get_ptr()).get_ptr());
 384                 if (result)
 385                     printf("Couldn't validate original archive with 7z!\n");
 386                 else
 387                 {
 388                     printf("Shuffling archive %s:\n", temp_filename.get_ptr());
 389                     if (!shuffle_order_of_archive(filename.get_ptr(), temp_filename.get_ptr()))
 390                         FAIL;
 391
 392                     printf("Validating with miniz_zip:\n");
 393                     if (!mz_zip_validate_file_archive(temp_filename.get_ptr(), 0, &last_zip_error))
 394                         FAIL;
 395
 396                     printf("Validating with 7z:\n");
 397                     int result = system(dynamic_string(cVarArg, "7z t \"%s\"", temp_filename.get_ptr()).get_ptr());
 398                     if (result)
 399                         FAIL;
 400
 401                     printf("Ok\n");
 402                 }
 403             }
 404         }
 405
 406         printf("Total failures: %u, total good: %u\n", num_failures, num_good);
 407         for (uint i = 0; i < failures.size(); i++)
 408             printf("%s\n", failures[i].get_ptr());
 409         return true;
 410     }
 411
 412     bool mz_zip_test()
 413     {
 414         g_thread_safe_random.seed(1000);
 415
 416         //CHECK(mz_zip_validate_file_archive("/home/richg/temp/blah.zip", 0, NULL));
 417         //CHECK(mz_zip_validate_file_archive("big.zip", 0, NULL));
 418         //CHECK(shuffle_order_of_archive("/home/richg/temp/blah.zip", "big.zip"));
 419
 420         //CHECK(shuffle_order_of_archive("big.zip", "big2.zip"));
 421         //CHECK(mz_zip_validate_file_archive("big2.zip", 0, NULL));
 422
 423         //return true;
 424
 425         CHECK(validate_archives_in_dir("/media/AB80-6D4D/*.zip"));
 426         return true;
 427
 428 //CHECK(validate_archives_in_dir("/home/richg/dev/raddebugger/bin/temp4/*.zip"));
 429 //return true;
 430
 431 //CHECK(validate_archives_in_dir("/media/Green/dev/*.zip"));
 432
 433 //CHECK(validate_archives_in_dir("/home/richg/temp/*.zip"));
 434
 435 //return true;
 436
 437 //CHECK(mz_zip_validate_file_archive("g1.zip", 0));
 438 //return true;
 439
 440 #if 0
 441         //void *p = calloc(0xFFFFFFFF, 1);
 442         //bool status = mz_zip_add_mem_to_archive_file_in_place("x2.zip", "1.bin", p, 0xFFFFFFFF, NULL, 0, 1, false);
 443         //CHECK(status);
 444
 445         CHECK(mz_zip_validate_file_archive("x2.zip", MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG));
 446
 447         CHECK(mz_zip_validate_file_archive("x3.zip", MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG));
 448
 449         CHECK(mz_zip_validate_file_archive("x.zip", MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG));
 450
 451         mz_zip_archive zip_reader;
 452         mz_zip_zero_struct(&zip_reader);
 453         CHECK(mz_zip_reader_init_file(&zip_reader, "x2.zip", 0));
 454
 455         mz_zip_archive zip;
 456         mz_zip_zero_struct(&zip);
 457
 458         CHECK(mz_zip_writer_init_file(&zip, "x3.zip", 0, MZ_ZIP_FLAG_WRITE_ZIP64));
 459         CHECK(mz_zip_writer_add_from_zip_reader(&zip, &zip_reader, 0));
 460         CHECK(mz_zip_writer_finalize_archive(&zip));
 461         CHECK(mz_zip_writer_end(&zip));
 462
 463         exit(0);
 464
 465         //bool success = validate_archive("z2.zip", true);
 466         //CHECK(success);
 467         //exit(0);
 468 #endif
 469
 470         random rm;
 471         rm.seed_from_urandom();
 472
 473         mz_zip_archive src_zip64;
 474         mz_zip_zero_struct(&src_zip64);
 475         CHECK(mz_zip_reader_init_file(&src_zip64, "/home/richg/temp/a.zip", 0, 0, 0));
 476         //CHECK(mz_zip_is_zip64(&src_zip64));
 477
 478         mz_zip_archive src_zip32;
 479         mz_zip_zero_struct(&src_zip32);
 480         CHECK(mz_zip_reader_init_file(&src_zip32, "/home/richg/temp/a.zip", 0, 0, 0));
 481         //CHECK(!mz_zip_is_zip64(&src_zip32));
 482
 483         cfile_stream rand_stream("/home/richg/temp/output1.bin", cDataStreamReadable | cDataStreamSeekable);
 484         if (!rand_stream.is_opened())
 485             return false;
 486
 487         int zip_base = rm.urand_inclusive('a', 'z');
 488         int start_seed = g_thread_safe_random.urand(1, 10000);
 489
 490         dynamic_string temp_filename(cVarArg, "temp_%u.bin", g_thread_safe_random.urand32());
 491
 492         for (uint t = 0; t < 10000; t++)
 493         {
 494             uint32 seed = t + start_seed;
 495             printf("******* Pass %u, seed %u\n", t, seed);
 496
 497             g_thread_safe_random.seed(seed);
 498
 499             dynamic_string zip_filename(cVarArg, "%c%u.zip", zip_base, t + 1);
 500
 501             dynamic_string full_zip_filename(zip_filename);
 502             file_utils::full_path(full_zip_filename);
 503
 504             bool zip64 = g_thread_safe_random.get_bit();
 505
 506             bool test_big_files = g_thread_safe_random.get_bit();
 507
 508             uint max_files;
 509             if (test_big_files)
 510                 max_files = (uint)g_thread_safe_random.urand64_inclusive(0, 10);
 511             else
 512                 max_files = (uint)g_thread_safe_random.urand64_inclusive(0, zip64 ? 200000 : 65535);
 513
 514             printf("Max files: %u, zip64: %u\n", max_files, zip64);
 515
 516             bool success;
 517
 518             uint64_t min_file_size = 0;
 519             uint64_t max_file_size = 1000000; //zip64 ? 0xFFFFFFFFFF : 0xFFFFFFFF;
 520             if (!test_big_files)
 521             {
 522                 max_file_size = 1024;
 523             }
 524
 525             zip_file_desc_vec file_descs;
 526             success = mz_zip_create_random_file_archive(rand_stream, zip_filename.get_ptr(), zip64, min_file_size, max_file_size, 0, max_files, file_descs, zip64 ? (g_thread_safe_random.get_bit() ? &src_zip64 : &src_zip32) : &src_zip32);
 527             CHECK(success);
 528
 529             uint num_actual_files = file_descs.size();
 530
 531             printf("******* Validating archive %s using miniz_zip\n", full_zip_filename.get_ptr());
 532
 533             mz_zip_error last_zip_error = MZ_ZIP_NO_ERROR;
 534             success = mz_zip_validate_file_archive(zip_filename.get_ptr(), ((g_thread_safe_random.get_bit() && (file_descs.size() < 30000)) ? MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY : 0) | MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG, &last_zip_error);
 535             CHECK(success);
 536
 537             printf("******* Validating archive %s using unzip\n", full_zip_filename.get_ptr());
 538
 539             int result = system(dynamic_string(cVarArg, "unzip -t \"%s\"", full_zip_filename.get_ptr()).get_ptr());
 540             if ((num_actual_files == 0) && (result == 256))
 541             {
 542             }
 543             else
 544             {
 545                 CHECK(result == 0);
 546             }
 547
 548             printf("******* Validating archive %s using 7z\n", full_zip_filename.get_ptr());
 549
 550             result = system(dynamic_string(cVarArg, "7z t \"%s\"", full_zip_filename.get_ptr()).get_ptr());
 551             CHECK(result == 0);
 552
 553             printf("Shuffling archive %s:\n", full_zip_filename.get_ptr());
 554             if (!shuffle_order_of_archive(full_zip_filename.get_ptr(), temp_filename.get_ptr()))
 555                 FAIL;
 556
 557             printf("Validating with miniz_zip:\n");
 558             if (!mz_zip_validate_file_archive(temp_filename.get_ptr(), 0, &last_zip_error))
 559                 FAIL;
 560
 561             printf("Validating with 7z:\n");
 562             result = system(dynamic_string(cVarArg, "7z t \"%s\"", temp_filename.get_ptr()).get_ptr());
 563             if (result)
 564                 FAIL;
 565         }
 566
 567         // this func potentially leaks if something fails
 568         mz_zip_reader_end(&src_zip32);
 569         mz_zip_reader_end(&src_zip64);
 570
 571         return true;
 572     }
 573
 574 } // namespace vogl