1 /**************************************************************************
3 * Copyright 2013-2014 RAD Game Tools and Valve Software
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 **************************************************************************/
26 // File: vogl_blob_manager.cpp
27 #include "vogl_common.h"
28 #include "vogl_blob_manager.h"
29 #include "vogl_buffer_stream.h"
30 #include "vogl_cfile_stream.h"
31 #include "vogl_file_utils.h"
32 #include "vogl_find_files.h"
33 #include "vogl_hash.h"
37 //----------------------------------------------------------------------------------------------------------------------
39 //----------------------------------------------------------------------------------------------------------------------
40 vogl_blob_manager::vogl_blob_manager()
47 vogl_blob_manager::~vogl_blob_manager()
54 dynamic_string vogl_blob_manager::compute_unique_id(const void *pData, uint size, const dynamic_string &prefix, const dynamic_string &ext, const uint64_t *pCRC64) const
58 VOGL_ASSERT(!prefix.contains('['));
59 VOGL_ASSERT(!prefix.contains(']'));
61 uint64_t crc64 = pCRC64 ? *pCRC64 : calc_crc64(CRC64_INIT, static_cast<const uint8 *>(pData), size);
63 dynamic_string actual_ext(ext);
64 if ((actual_ext.get_len() > 1) && (actual_ext[0] == '.'))
68 return dynamic_string(cVarArg, "[%s]_%" PRIX64 "_%u.radblob.%s", prefix.get_ptr(), crc64, size, actual_ext.get_len() ? actual_ext.get_ptr() : "raw");
70 return dynamic_string(cVarArg, "%" PRIX64 "_%u.radblob.%s", crc64, size, actual_ext.get_len() ? actual_ext.get_ptr() : "raw");
73 dynamic_string vogl_blob_manager::get_prefix(const dynamic_string &id) const
77 if (!id.is_empty() && id.begins_with("["))
79 dynamic_string prefix(id);
80 int e = prefix.find_right(']');
81 return prefix.mid(1, e - 1);
86 dynamic_string vogl_blob_manager::get_extension(const dynamic_string &id) const
90 dynamic_string extension(id);
91 file_utils::get_extension(extension);
95 bool vogl_blob_manager::get(const dynamic_string &id, uint8_vec &data) const
99 if (!is_initialized())
105 data_stream *pStream = open(id);
109 vogl_error_printf("%s: Failed finding blob ID %s\n", VOGL_METHOD_NAME, id.get_ptr());
114 if (pStream->get_size() > static_cast<uint64_t>(cINT32_MAX))
120 vogl_error_printf("%s: Blob is too large: blob ID %s, size %" PRIu64 "\n", VOGL_METHOD_NAME, id.get_ptr(), pStream->get_size());
125 uint32 size = static_cast<uint32>(pStream->get_size());
127 if (!data.try_resize(size))
130 vogl_error_printf("%s: Out of memory while trying to read blob ID %s, size %u\n", VOGL_METHOD_NAME, id.get_ptr(), size);
136 if (pStream->read(data.get_ptr(), size) != size)
142 vogl_error_printf("%s: Failed reading blob ID %s, size %u\n", VOGL_METHOD_NAME, id.get_ptr(), size);
152 bool vogl_blob_manager::populate(const vogl_blob_manager &other)
156 if (!is_initialized() || !other.is_initialized())
162 dynamic_string_array ids(other.enumerate());
165 for (uint i = 0; i < ids.size(); i++)
167 const dynamic_string &id = ids[i];
169 data_stream *pStream = other.open(id);
176 dynamic_string new_id(add_stream_using_id(*pStream, id));
177 if (new_id.is_empty())
180 other.close(pStream);
186 dynamic_string vogl_blob_manager::add_buf_compute_unique_id(const void *pData, uint size, const vogl::dynamic_string &prefix, const dynamic_string &ext, const uint64_t *pCRC64)
190 if (!is_initialized() || !is_writable())
193 dynamic_string id(compute_unique_id(pData, size, prefix, ext, pCRC64));
195 return add_buf_using_id(pData, size, id);
198 dynamic_string vogl_blob_manager::add_stream_compute_unique_id(data_stream &stream, const dynamic_string &prefix, const dynamic_string &ext, const uint64_t *pCRC64)
202 if (!is_initialized() || !is_writable())
209 if (stream.get_size() > static_cast<uint64_t>(cUINT32_MAX))
215 uint64_t size64 = stream.get_size();
218 if ((size64 > static_cast<uint64_t>(VOGL_MAX_POSSIBLE_HEAP_BLOCK_SIZE)) || (size64 > cUINT32_MAX))
224 uint size = static_cast<uint>(size64);
226 void *pData = vogl_malloc(static_cast<size_t>(size));
230 if (!stream.seek(0, false))
236 if (stream.read64(pData, size) != size)
242 dynamic_string id(compute_unique_id(pData, size, prefix, ext, pCRC64));
244 dynamic_string actual_id(add_buf_using_id(pData, size, id));
250 dynamic_string vogl_blob_manager::add_stream_using_id(data_stream &stream, const dynamic_string &id)
254 if (!is_initialized() || !is_writable())
260 uint64_t size64 = stream.get_size();
263 if ((size64 > static_cast<uint64_t>(VOGL_MAX_POSSIBLE_HEAP_BLOCK_SIZE)) || (size64 > cUINT32_MAX))
269 uint size = static_cast<uint>(size64);
271 void *pData = vogl_malloc(static_cast<size_t>(size));
275 if (!stream.seek(0, false))
281 if (stream.read64(pData, size) != size)
287 dynamic_string actual_id(add_buf_using_id(pData, size, id));
293 //----------------------------------------------------------------------------------------------------------------------
294 // vogl_blob_manager::copy_file
295 //----------------------------------------------------------------------------------------------------------------------
296 vogl::dynamic_string vogl_blob_manager::copy_file(vogl_blob_manager &src_blob_manager, const vogl::dynamic_string &src_id, const vogl::dynamic_string &dst_id)
299 if (!src_blob_manager.get(src_id, data))
302 return add_buf_using_id(data.get_ptr(), data.size(), dst_id);
305 //----------------------------------------------------------------------------------------------------------------------
306 // vogl_memory_blob_manager
307 //----------------------------------------------------------------------------------------------------------------------
308 vogl_memory_blob_manager::vogl_memory_blob_manager()
309 : vogl_blob_manager()
314 vogl_memory_blob_manager::~vogl_memory_blob_manager()
321 bool vogl_memory_blob_manager::init(uint32 flags)
327 if (!vogl_blob_manager::init(flags))
330 m_initialized = true;
334 bool vogl_memory_blob_manager::deinit()
339 return vogl_blob_manager::deinit();
342 dynamic_string vogl_memory_blob_manager::add_buf_using_id(const void *pData, uint size, const dynamic_string &id)
346 if (!is_initialized() || !is_writable())
352 dynamic_string actual_id(id);
353 if (actual_id.is_empty())
354 actual_id = compute_unique_id(pData, size);
356 blob_map::insert_result insert_res(m_blobs.insert(actual_id));
357 if (!insert_res.second)
358 return (insert_res.first)->first;
360 blob &new_blob = (insert_res.first)->second;
361 new_blob.m_id = actual_id;
362 new_blob.m_blob.append(static_cast<const uint8 *>(pData), static_cast<uint32>(size));
367 data_stream *vogl_memory_blob_manager::open(const dynamic_string &id) const
371 if (!is_initialized() || !is_readable())
377 const blob *pBlob = m_blobs.find_value(id);
381 return vogl_new(buffer_stream, pBlob->m_blob.get_ptr(), pBlob->m_blob.size());
384 void vogl_memory_blob_manager::close(data_stream *pStream) const
388 vogl_delete(pStream);
391 bool vogl_memory_blob_manager::does_exist(const dynamic_string &id) const
395 return m_blobs.contains(id);
398 uint64_t vogl_memory_blob_manager::get_size(const dynamic_string &id) const
402 const blob *pBlob = m_blobs.find_value(id);
403 return pBlob ? pBlob->m_blob.size() : 0;
406 dynamic_string_array vogl_memory_blob_manager::enumerate() const
410 if (!is_initialized())
413 return dynamic_string_array();
416 dynamic_string_array ids;
417 return m_blobs.get_keys(ids);
420 //----------------------------------------------------------------------------------------------------------------------
421 // vogl_file_blob_manager
422 //----------------------------------------------------------------------------------------------------------------------
424 vogl_loose_file_blob_manager::vogl_loose_file_blob_manager()
425 : vogl_blob_manager()
430 vogl_loose_file_blob_manager::~vogl_loose_file_blob_manager()
437 bool vogl_loose_file_blob_manager::init(uint32 flags)
443 if (!vogl_blob_manager::init(flags))
445 m_initialized = true;
449 bool vogl_loose_file_blob_manager::init(uint32 flags, const char *pPath)
455 if (!vogl_blob_manager::init(flags))
458 m_initialized = true;
462 bool vogl_loose_file_blob_manager::deinit()
466 return vogl_blob_manager::deinit();
469 dynamic_string vogl_loose_file_blob_manager::add_buf_using_id(const void *pData, uint size, const dynamic_string &id)
479 dynamic_string actual_id(id);
480 if (actual_id.is_empty())
481 actual_id = compute_unique_id(pData, size);
483 dynamic_string filename(get_filename(actual_id));
485 if (does_exist(filename))
487 uint64_t cur_size = get_size(actual_id);
488 if (cur_size != size)
489 vogl_error_printf("%s: Not overwrite already existing blob %s desired size %u, but it has the wrong size on disk (%" PRIu64 " bytes)!\n", VOGL_METHOD_NAME, filename.get_ptr(), size, cur_size);
491 vogl_message_printf("%s: Not overwriting already existing blob %s size %u\n", VOGL_METHOD_NAME, filename.get_ptr(), size);
495 cfile_stream out_file(filename.get_ptr(), cDataStreamWritable);
496 if (!out_file.is_opened())
498 vogl_error_printf("%s: Failed creating file \"%s\"!\n", VOGL_METHOD_NAME, filename.get_ptr());
502 if (out_file.write(pData, size) != size)
507 file_utils::delete_file(filename.get_ptr());
509 vogl_error_printf("%s: Failed writing to file \"%s\"!\n", VOGL_METHOD_NAME, filename.get_ptr());
514 if (!out_file.close())
518 file_utils::delete_file(filename.get_ptr());
520 vogl_error_printf("%s: Failed writing to file \"%s\"!\n", VOGL_METHOD_NAME, filename.get_ptr());
528 data_stream *vogl_loose_file_blob_manager::open(const dynamic_string &id) const
538 cfile_stream *pStream = vogl_new(cfile_stream, get_filename(id).get_ptr());
539 if (!pStream->is_opened())
541 vogl_delete(pStream);
547 void vogl_loose_file_blob_manager::close(data_stream *pStream) const
551 vogl_delete(pStream);
554 bool vogl_loose_file_blob_manager::does_exist(const dynamic_string &id) const
558 if (!is_initialized())
564 return file_utils::does_file_exist(get_filename(id).get_ptr());
567 uint64_t vogl_loose_file_blob_manager::get_size(const dynamic_string &id) const
571 if (!is_initialized())
578 file_utils::get_file_size(get_filename(id).get_ptr(), file_size);
582 dynamic_string vogl_loose_file_blob_manager::get_filename(const dynamic_string &id) const
586 if (!is_initialized())
592 dynamic_string file_path;
593 file_utils::combine_path(file_path, m_path.get_ptr(), id.get_ptr());
597 dynamic_string_array vogl_loose_file_blob_manager::enumerate() const
601 dynamic_string_array files;
603 if (!is_initialized())
609 dynamic_string find_path(get_filename("*.radblob.*"));
613 bool success = finder.find(find_path.get_ptr());
616 const find_files::file_desc_vec &found_files = finder.get_files();
618 for (uint i = 0; i < found_files.size(); i++)
619 files.push_back(found_files[i].m_name);
625 //----------------------------------------------------------------------------------------------------------------------
626 // vogl_archive_blob_manager
627 //----------------------------------------------------------------------------------------------------------------------
628 vogl_archive_blob_manager::vogl_archive_blob_manager()
629 : vogl_blob_manager()
633 mz_zip_zero_struct(&m_zip);
636 vogl_archive_blob_manager::~vogl_archive_blob_manager()
643 bool vogl_archive_blob_manager::init_memory(uint32 flags, const void *pZip_data, size_t size)
649 if (!vogl_blob_manager::init(flags))
652 if (is_writable() || !is_readable())
658 if (!mz_zip_reader_init_mem(&m_zip, pZip_data, size, 0))
660 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
661 vogl_error_printf("%s: mz_zip_reader_init_mem() failed initing memory size %" PRIu64 ", error 0x%X (%s)\n", VOGL_METHOD_NAME, cast_val_to_uint64(size), mz_err, mz_zip_get_error_string(mz_err));
669 m_initialized = true;
674 bool vogl_archive_blob_manager::init_heap(uint32 flags)
680 if (!vogl_blob_manager::init(flags))
689 if (!mz_zip_writer_init_heap(&m_zip, 0, 1024, MZ_ZIP_FLAG_WRITE_ALLOW_READING | MZ_ZIP_FLAG_WRITE_ZIP64))
691 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
692 vogl_error_printf("%s: mz_zip_writer_init_heap() failed initing heap with flags 0x%x, error 0x%X (%s)\n", VOGL_METHOD_NAME, flags, mz_err, mz_zip_get_error_string(mz_err));
698 m_initialized = true;
703 void *vogl_archive_blob_manager::deinit_heap(size_t &size)
709 if ((mz_zip_get_mode(&m_zip) == MZ_ZIP_MODE_INVALID) || (mz_zip_get_type(&m_zip) != MZ_ZIP_TYPE_HEAP))
713 if (!mz_zip_writer_finalize_heap_archive(&m_zip, &pBuf, &size))
715 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
716 vogl_error_printf("%s: mz_zip_writer_finalize_heap_archive() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
726 bool vogl_archive_blob_manager::init_file(uint32 flags, const char *pFilename, uint64_t file_start_ofs, uint64_t actual_archive_size)
732 if (!vogl_blob_manager::init(flags))
735 m_archive_filename = pFilename;
739 // open existing and read/append
741 if (is_writable() && (flags & (cBMFOpenExisting | cBMFOpenExistingOrCreateNew)))
743 bool does_file_exist = file_utils::does_file_exist(pFilename);
745 // open existing for read/write
748 if (!mz_zip_reader_init_file(&m_zip, pFilename, 0, file_start_ofs, actual_archive_size))
750 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
751 vogl_error_printf("%s: mz_zip_reader_init_file() failed with filename \"%s\", error 0x%X (%s)\n", VOGL_METHOD_NAME, pFilename, mz_err, mz_zip_get_error_string(mz_err));
757 if (!mz_zip_writer_init_from_reader(&m_zip, pFilename, MZ_ZIP_FLAG_WRITE_ZIP64))
759 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
760 vogl_error_printf("%s: mz_zip_writer_init_from_reader() failed with filename \"%s\", error 0x%X (%s)\n", VOGL_METHOD_NAME, pFilename, mz_err, mz_zip_get_error_string(mz_err));
768 // File does not exist - check if the caller allows us to create a new archive.
769 if ((flags & cBMFOpenExistingOrCreateNew) == 0)
775 // Create new archive, write only or read/write.
776 if (!mz_zip_writer_init_file(&m_zip, pFilename, file_start_ofs, (is_readable() ? MZ_ZIP_FLAG_WRITE_ALLOW_READING : 0) | MZ_ZIP_FLAG_WRITE_ZIP64))
778 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
779 vogl_error_printf("%s: mz_zip_writer_init_file() failed with filename \"%s\", error 0x%X (%s)\n", VOGL_METHOD_NAME, pFilename, mz_err, mz_zip_get_error_string(mz_err));
786 else if (is_read_only())
788 // open existing for reading
789 if (flags & cBMFOpenExistingOrCreateNew)
795 if (!mz_zip_reader_init_file(&m_zip, pFilename, 0, file_start_ofs, actual_archive_size))
797 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
798 vogl_error_printf("%s: mz_zip_reader_init_file() failed with filename \"%s\", error 0x%X (%s)\n", VOGL_METHOD_NAME, pFilename, mz_err, mz_zip_get_error_string(mz_err));
806 VOGL_ASSERT(!actual_archive_size);
808 // create new archive, read/write or write only
809 if (!mz_zip_writer_init_file(&m_zip, pFilename, file_start_ofs, (is_readable() ? MZ_ZIP_FLAG_WRITE_ALLOW_READING : 0) | MZ_ZIP_FLAG_WRITE_ZIP64))
811 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
812 vogl_error_printf("%s: mz_zip_writer_init_file() failed with filename \"%s\", error 0x%X (%s)\n", VOGL_METHOD_NAME, pFilename, mz_err, mz_zip_get_error_string(mz_err));
819 if (!populate_blob_map())
825 m_initialized = true;
830 bool vogl_archive_blob_manager::init_file_temp(uint32 flags, const char *pPath)
834 dynamic_string temp_filename(file_utils::generate_temp_filename(pPath));
835 return init_file(flags, temp_filename.get_ptr());
838 bool vogl_archive_blob_manager::init_cfile(uint32 flags, FILE *pFile, uint64_t cur_size)
844 if (!vogl_blob_manager::init(flags))
847 m_archive_filename = "<cfile>";
851 // Writable blob manager - see if there's any existing archive data
854 if (!mz_zip_reader_init_cfile(&m_zip, pFile, cur_size, 0))
856 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
857 vogl_error_printf("%s: mz_zip_reader_init_cfile() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
863 if (!mz_zip_writer_init_from_reader(&m_zip, NULL, MZ_ZIP_FLAG_WRITE_ZIP64))
865 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
866 vogl_error_printf("%s: mz_zip_writer_init_from_reader() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
874 // create new archive, write only or read/write
875 if (!mz_zip_writer_init_cfile(&m_zip, pFile, (is_readable() ? MZ_ZIP_FLAG_WRITE_ALLOW_READING : 0) | MZ_ZIP_FLAG_WRITE_ZIP64))
877 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
878 vogl_error_printf("%s: mz_zip_writer_init_cfile() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
885 else if (is_read_only())
887 // Open existing read-only archive
888 if (!mz_zip_reader_init_cfile(&m_zip, pFile, cur_size, 0))
890 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
891 vogl_error_printf("%s: mz_zip_reader_init_cfile() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
899 // create new archive, read/write or write only
906 if (!mz_zip_writer_init_cfile(&m_zip, pFile, (is_readable() ? MZ_ZIP_FLAG_WRITE_ALLOW_READING : 0) | MZ_ZIP_FLAG_WRITE_ZIP64))
908 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
909 vogl_error_printf("%s: mz_zip_writer_init_cfile() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
916 if (!populate_blob_map())
925 bool vogl_archive_blob_manager::populate_blob_map()
931 for (uint file_index = 0; file_index < mz_zip_get_num_files(&m_zip); file_index++)
933 mz_zip_archive_file_stat stat;
934 if (!mz_zip_file_stat(&m_zip, file_index, &stat))
936 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
937 vogl_error_printf("%s: mz_zip_file_stat() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
942 if (!m_blobs.insert(stat.m_filename, blob(stat.m_filename, file_index, stat.m_uncomp_size)).second)
944 vogl_warning_printf("%s: Duplicate file %s in blob archive %s\n", VOGL_METHOD_NAME, stat.m_filename, m_archive_filename.get_ptr());
951 bool vogl_archive_blob_manager::deinit()
956 VOGL_NOTE_UNUSED(status);
958 if (mz_zip_get_mode(&m_zip) != MZ_ZIP_MODE_INVALID)
960 if ((mz_zip_get_type(&m_zip) == MZ_ZIP_TYPE_FILE) && (mz_zip_get_mode(&m_zip) == MZ_ZIP_MODE_WRITING))
962 if (!mz_zip_writer_finalize_archive(&m_zip))
964 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
965 vogl_error_printf("%s: mz_zip_writer_finalize_archive() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
971 if (!mz_zip_end(&m_zip))
973 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
974 vogl_error_printf("%s: mz_zip_end() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
980 mz_zip_zero_struct(&m_zip);
982 m_archive_filename.clear();
986 return vogl_blob_manager::deinit();
989 vogl::dynamic_string vogl_archive_blob_manager::add_buf_using_id(const void *pData, uint size, const vogl::dynamic_string &id)
993 if (!is_initialized() || !is_writable())
999 if (mz_zip_get_mode(&m_zip) != MZ_ZIP_MODE_WRITING)
1002 dynamic_string actual_id(id);
1003 if (actual_id.is_empty())
1004 actual_id = compute_unique_id(pData, size);
1006 // We don't support overwriting files already in the archive - it's up to the caller to not try adding redundant files into the archive.
1007 // We could support orphaning the previous copy of the file and updating the archive to point to the latest version, though.
1008 if (m_blobs.contains(actual_id))
1010 vogl_debug_printf("%s: Archive already contains blob id \"%s\"! Not replacing file.\n", VOGL_METHOD_NAME, actual_id.get_ptr());
1014 uint file_index = mz_zip_get_num_files(&m_zip);
1016 // TODO: Allow caller to control whether files are compressed
1017 if (!mz_zip_writer_add_mem(&m_zip, actual_id.get_ptr(), pData, size, MZ_BEST_SPEED))
1018 //if (!mz_zip_writer_add_mem(&m_zip, actual_id.get_ptr(), pData, size, 0))
1020 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
1021 vogl_error_printf("%s: mz_zip_writer_add_mem() failed adding blob \"%s\" size %u, error 0x%X (%s)\n", VOGL_METHOD_NAME, id.get_ptr(), size, mz_err, mz_zip_get_error_string(mz_err));
1026 bool success = m_blobs.insert(actual_id, blob(actual_id, file_index, size)).second;
1027 VOGL_NOTE_UNUSED(success);
1028 VOGL_ASSERT(success);
1033 vogl::data_stream *vogl_archive_blob_manager::open(const vogl::dynamic_string &id) const
1037 if (!is_initialized() || !is_readable())
1043 blob_map::const_iterator it = m_blobs.find(id);
1044 if (it == m_blobs.end())
1047 // TODO: Add some sort of streaming decompression support to miniz and this class.
1049 mz_zip_clear_last_error(&m_zip);
1052 void *pBuf = mz_zip_extract_to_heap(&m_zip, it->second.m_file_index, &size, 0);
1055 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
1056 vogl_error_printf("%s: mz_zip_extract_to_heap() failed opening blob \"%s\", error 0x%X (%s)\n", VOGL_METHOD_NAME, id.get_ptr(), mz_err, mz_zip_get_error_string(mz_err));
1061 VOGL_VERIFY(size == it->second.m_size);
1063 return vogl_new(vogl::buffer_stream, pBuf, size);
1066 void vogl_archive_blob_manager::close(vogl::data_stream *pStream) const
1072 void *pBuf = const_cast<void *>(pStream->get_ptr());
1074 vogl_delete(pStream);
1080 bool vogl_archive_blob_manager::does_exist(const vogl::dynamic_string &id) const
1084 if (!is_initialized())
1090 return m_blobs.contains(id);
1093 uint64_t vogl_archive_blob_manager::get_size(const vogl::dynamic_string &id) const
1097 if (!is_initialized())
1103 blob_map::const_iterator it = m_blobs.find(id);
1104 if (it == m_blobs.end())
1107 return it->second.m_size;
1110 vogl::dynamic_string_array vogl_archive_blob_manager::enumerate() const
1114 if (!is_initialized())
1117 return vogl::dynamic_string_array();
1120 vogl::dynamic_string_array result(m_blobs.size());
1123 for (blob_map::const_iterator it = m_blobs.begin(); it != m_blobs.end(); ++it)
1124 result[index++] = it->first;
1129 uint64_t vogl_archive_blob_manager::get_archive_size() const
1133 if (!is_initialized())
1136 return mz_zip_get_archive_size(&m_zip);
1139 bool vogl_archive_blob_manager::write_archive_to_stream(vogl::data_stream &stream) const
1143 if (!is_initialized())
1146 uint8_vec buf(64 * 1024);
1148 uint64_t bytes_remaining = mz_zip_get_archive_size(&m_zip);
1149 mz_uint64 src_file_ofs = 0;
1151 while (bytes_remaining)
1153 size_t n = static_cast<size_t>(math::minimum<uint64_t>(bytes_remaining, buf.size()));
1155 if (mz_zip_read_archive_data(&m_zip, src_file_ofs, buf.get_ptr(), n) != n)
1157 mz_zip_error mz_err = mz_zip_get_last_error(&m_zip);
1158 vogl_error_printf("%s: mz_zip_read_archive_data() failed, error 0x%X (%s)\n", VOGL_METHOD_NAME, mz_err, mz_zip_get_error_string(mz_err));
1163 if (stream.write(buf.get_ptr(), static_cast<uint>(n)) != n)
1167 bytes_remaining -= n;
1173 //----------------------------------------------------------------------------------------------------------------------
1174 // vogl_multi_blob_manager
1175 //----------------------------------------------------------------------------------------------------------------------
1176 vogl_multi_blob_manager::vogl_multi_blob_manager()
1181 vogl_multi_blob_manager::~vogl_multi_blob_manager()
1186 bool vogl_multi_blob_manager::init(uint32 flags)
1192 if (flags & (cBMFWritable | cBMFOpenExistingOrCreateNew))
1195 if (!vogl_blob_manager::init(flags))
1198 m_initialized = true;
1203 void vogl_multi_blob_manager::add_blob_manager(vogl_blob_manager *pBlob_manager)
1207 if (!is_initialized())
1213 m_blob_managers.push_back(pBlob_manager);
1216 void vogl_multi_blob_manager::remove_blob_manager(vogl_blob_manager *pBlob_manager)
1220 if (!is_initialized())
1226 int index = m_blob_managers.find(pBlob_manager);
1228 m_blob_managers.erase(index);
1231 bool vogl_multi_blob_manager::deinit()
1235 m_blob_managers.clear();
1237 return vogl_blob_manager::deinit();
1240 vogl::dynamic_string vogl_multi_blob_manager::add_buf_using_id(const void *pData, uint size, const vogl::dynamic_string &id)
1245 VOGL_NOTE_UNUSED(pData);
1246 VOGL_NOTE_UNUSED(size);
1247 VOGL_NOTE_UNUSED(id);
1251 vogl::data_stream *vogl_multi_blob_manager::open(const dynamic_string &id) const
1255 if (!is_initialized())
1261 for (uint i = 0; i < m_blob_managers.size(); i++)
1263 if (!m_blob_managers[i]->is_initialized())
1266 vogl::data_stream *pStream = m_blob_managers[i]->open(id);
1269 VOGL_ASSERT(!pStream->get_user_data());
1270 pStream->set_user_data(m_blob_managers[i]);
1278 void vogl_multi_blob_manager::close(vogl::data_stream *pStream) const
1284 vogl_blob_manager *pBlob_manager = static_cast<vogl_blob_manager *>(pStream->get_user_data());
1285 VOGL_ASSERT(pBlob_manager);
1289 pStream->set_user_data(NULL);
1290 pBlob_manager->close(pStream);
1295 bool vogl_multi_blob_manager::does_exist(const vogl::dynamic_string &id) const
1299 for (uint i = 0; i < m_blob_managers.size(); i++)
1301 if (!m_blob_managers[i]->is_initialized())
1304 if (m_blob_managers[i]->does_exist(id))
1310 uint64_t vogl_multi_blob_manager::get_size(const vogl::dynamic_string &id) const
1314 for (uint i = 0; i < m_blob_managers.size(); i++)
1316 if (!m_blob_managers[i]->is_initialized())
1319 if (m_blob_managers[i]->does_exist(id))
1320 return m_blob_managers[i]->get_size(id);
1325 vogl::dynamic_string_array vogl_multi_blob_manager::enumerate() const
1329 vogl::dynamic_string_array all_files;
1331 for (uint i = 0; i < m_blob_managers.size(); i++)
1333 if (!m_blob_managers[i]->is_initialized())
1336 vogl::dynamic_string_array files(m_blob_managers[i]->enumerate());
1337 all_files.append(files);