asset_manager.cpp 14 KB
Newer Older
1
2
#include <mirrage/asset/asset_manager.hpp>

3
#include <mirrage/asset/embedded_asset.hpp>
4
5
#include <mirrage/asset/error.hpp>

6
#include <mirrage/utils/log.hpp>
7
#include <mirrage/utils/md5.hpp>
8
#include <mirrage/utils/template_utils.hpp>
9
10
11
12

#include <physfs.h>

#include <cstdio>
13
#include <cstring>
14

Florian Oetke's avatar
Florian Oetke committed
15
#ifdef _WIN32
16
17
#include <direct.h>
#include <windows.h>
18
#else
19
20
#include <sys/stat.h>
#include <unistd.h>
21
22
23
#endif


24
using namespace mirrage::util;
25
using namespace std::string_literals;
26
27
28

namespace {

29
30
	std::string append_file(const std::string& folder, const std::string file)
	{
31
		if(ends_with(folder, "/") || starts_with(file, "/"))
32
			return folder + file;
33
		else
34
			return folder + "/" + file;
35
	}
36
37
	void create_dir(const std::string& dir)
	{
Florian Oetke's avatar
Florian Oetke committed
38
#ifdef _WIN32
39
40
41
42
43
44
		CreateDirectory(dir.c_str(), NULL);
#else
		mkdir(dir.c_str(), 0777);
#endif
	}

45
46
	auto last_of(const std::string& str, char c)
	{
47
		auto idx = str.find_last_of(c);
48
		return idx != std::string::npos ? mirrage::util::just(idx + 1) : mirrage::util::nothing;
49
50
	}

51
52
	auto split_path(const std::string& path)
	{
53
		auto filename_delim_end = last_of(path, '/').get_or(last_of(path, '\\').get_or(0));
54

55
		return std::make_tuple(path.substr(0, filename_delim_end - 1),
56
57
58
59
		                       path.substr(filename_delim_end, std::string::npos));
	}


60
61
	std::vector<std::string> list_files(const std::string& dir,
	                                    const std::string& prefix,
62
63
	                                    const std::string& suffix) noexcept
	{
64
65
		std::vector<std::string> res;

66
		char** rc = PHYSFS_enumerateFiles(dir.c_str());
67

68
		for(char** i = rc; *i != nullptr; i++) {
69
70
			std::string str(*i);

71
72
			if((prefix.length() == 0 || str.find(prefix) == 0)
			   && (suffix.length() == 0 || str.find(suffix) == str.length() - suffix.length()))
73
74
75
76
77
78
79
				res.emplace_back(std::move(str));
		}

		PHYSFS_freeList(rc);

		return res;
	}
80
81
	std::vector<std::string> list_wildcard_files(const std::string& wildcard_path)
	{
82
		auto   spr  = split_path(wildcard_path);
83
84
85
		auto&& path = std::get<0>(spr);
		auto&& file = std::get<1>(spr);

86
		auto wildcard = last_of(file, '*').get_or(file.length());
87
88
		LOG_IF(plog::warning, wildcard != file.find_first_of('*') + 1)
		        << "More than one wildcard ist currently not supported. Found in: " << wildcard_path;
89

90
91
		auto prefix = file.substr(0, wildcard - 1);
		auto suffix = wildcard < file.length() ? file.substr(0, wildcard - 1) : std::string();
92
93
94
95
96
97
98
99
100

		auto files = list_files(path, prefix, suffix);
		for(auto& file : files) {
			file = path + "/" + file;
		}

		return files;
	}

101
102
	bool exists_file(const std::string path)
	{
103
		if(!PHYSFS_exists(path.c_str()))
104
105
106
			return false;

		auto stat = PHYSFS_Stat{};
107
		if(!PHYSFS_stat(path.c_str(), &stat))
108
109
110
			return false;

		return stat.filetype == PHYSFS_FILETYPE_REGULAR;
111
	}
112
113
	bool exists_dir(const std::string path)
	{
114
		if(!PHYSFS_exists(path.c_str()))
115
116
117
			return false;

		auto stat = PHYSFS_Stat{};
118
		if(!PHYSFS_stat(path.c_str(), &stat))
119
120
121
			return false;

		return stat.filetype == PHYSFS_FILETYPE_DIRECTORY;
122
123
	}

124
	template <typename Callback>
125
126
	void print_dir_recursiv(const std::string& dir, uint8_t depth, Callback&& callback)
	{
127
		std::string p;
128
129
		for(uint8_t i = 0; i < depth; i++)
			p += "  ";
130

131
		callback(p + dir);
132
133
		depth++;
		for(auto&& f : list_files(dir, "", "")) {
134
			if(depth >= 5)
135
				callback(p + "  " + f);
136
			else
137
138
139
140
				print_dir_recursiv(f, depth, callback);
		}
	}

141
	void init_physicsfs(const std::string& exe_name, mirrage::util::maybe<std::string> additional_search_path)
142
	{
143
144
145
146
		if(PHYSFS_isInit())
			return;

		if(!PHYSFS_init(exe_name.empty() ? nullptr : exe_name.c_str())) {
147
148
			throw std::system_error(static_cast<mirrage::asset::Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to initalize PhysicsFS.");
149
		}
150
151
152
153
154
155

		if(!PHYSFS_mount(PHYSFS_getBaseDir(), nullptr, 1)
		   || !PHYSFS_mount(append_file(PHYSFS_getBaseDir(), "..").c_str(), nullptr, 1)
		   || !PHYSFS_mount(mirrage::asset::pwd().c_str(), nullptr, 1))
			throw std::system_error(static_cast<mirrage::asset::Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to setup default search path.");
156
157

		additional_search_path.process([&](auto& dir) { PHYSFS_mount(dir.c_str(), nullptr, 1); });
158
159
	}

160
	constexpr auto default_source = {std::make_tuple("assets", false), std::make_tuple("assets.zip", true)};
161
} // namespace
162

163
namespace mirrage::asset {
164

165
166
	std::string pwd()
	{
167
168
		char cCurrentPath[FILENAME_MAX];

169
#ifdef _WIN32
170
171
		_getcwd(cCurrentPath, sizeof(cCurrentPath));
#else
172
173
174
		if(getcwd(cCurrentPath, sizeof(cCurrentPath)) == nullptr) {
			MIRRAGE_FAIL("getcwd with max length " << FILENAME_MAX << " failed with error code " << errno);
		}
175
#endif
176
177
178

		return cCurrentPath;
	}
179
180
181
182
	std::string write_dir(const std::string&       exe_name,
	                      const std::string&       org_name,
	                      const std::string&       app_name,
	                      util::maybe<std::string> additional_search_path)
183
	{
184
		init_physicsfs(exe_name, additional_search_path);
185

186
		if(exists_dir("write_dir")) {
187
			return std::string(PHYSFS_getRealDir("write_dir")) + "/write_dir";
188
		}
189

190
		return PHYSFS_getPrefDir(org_name.c_str(), app_name.c_str());
191
	}
192

193
194
195
196
	Asset_manager::Asset_manager(const std::string&       exe_name,
	                             const std::string&       org_name,
	                             const std::string&       app_name,
	                             util::maybe<std::string> additional_search_path)
197
	{
198
		init_physicsfs(exe_name, additional_search_path);
199

200
		auto write_dir = ::mirrage::asset::write_dir(exe_name, org_name, app_name, additional_search_path);
201
		create_dir(write_dir);
202
		LOG(plog::debug) << "Write dir: " << write_dir;
203

204
		if(!PHYSFS_mount(write_dir.c_str(), nullptr, 0))
205
206
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to construct search path.");
207

208
		if(!PHYSFS_setWriteDir(write_dir.c_str()))
209
210
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to set write-dir: "s + write_dir);
211

212
213
214
215
216
217
		auto add_source = [](std::string path) {
			auto apath = PHYSFS_getRealDir(path.c_str());
			if(apath) {
				path = std::string(apath) + "/" + path;
			}

218
			LOG(plog::info) << "Added FS directory: " << path;
219
			if(!PHYSFS_mount(path.c_str(), nullptr, 1))
220
221
				throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
				                        "Error adding custom archive: "s + path);
222
223
		};

224
		if(!exists_file("archives.lst")) {
225
226
227
			bool lost = true;
			for(auto& s : default_source) {
				const char* path;
228
				bool        file;
229
230
231
232
233
234
235
236
237
238

				std::tie(path, file) = s;

				if(file ? exists_file(path) : exists_dir(path)) {
					add_source(path);
					lost = false;
				}
			}

			if(lost) {
239
240
				LOG(plog::fatal) << "No archives.lst found. printing search-path...\n";
				print_dir_recursiv("/", 0, [](auto&& path) { LOG(plog::fatal) << path; });
241

242
243
				throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
				                        "No archives.lst found.");
244
			} else {
245
				LOG(plog::info) << "No archives.lst found. Using defaults.";
246
247
			}
		} else {
248
249
			auto in = _open("cfg:archives.lst"_aid, "archives.lst");

250
			// load other archives
251
252
253
254
			for(auto&& l : in.lines()) {
				if(l.find_last_of('*') != std::string::npos) {
					for(auto& file : list_wildcard_files(l)) {
						add_source(file.c_str());
255
					}
256
					continue;
257
				}
258
259
				add_source(l.c_str());
			}
260
261
		}

Florian Oetke's avatar
Florian Oetke committed
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
		for(auto ea : Embedded_asset::instances()) {
			LOG(plog::info) << "Include embedded asset \"" << ea->name() << "\": " << ea->data().size()
			                << " bytes MD5: "
			                << util::md5(std::string(reinterpret_cast<const char*>(ea->data().data()),
			                                         std::size_t(ea->data().size_bytes())));
			auto name = "embedded_" + ea->name() + ".zip";
			if(!PHYSFS_mountMemory(ea->data().data(),
			                       static_cast<PHYSFS_uint64>(ea->data().size_bytes()),
			                       nullptr,
			                       name.c_str(),
			                       nullptr,
			                       1)) {
				throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
				                        "Unable to add embedded archive: "s + ea->name());
			}
		}

279
280
281
282
		// unmount default search-path
		PHYSFS_unmount(PHYSFS_getBaseDir());
		PHYSFS_unmount(append_file(PHYSFS_getBaseDir(), "..").c_str());
		PHYSFS_unmount(mirrage::asset::pwd().c_str());
283
		additional_search_path.process([&](auto& dir) { PHYSFS_unmount(dir.c_str()); });
284

285
286
287
		_reload_dispatchers();
	}

288
289
	Asset_manager::~Asset_manager()
	{
290
291
		_containers.clear();
		if(!PHYSFS_deinit()) {
292
293
			MIRRAGE_FAIL(
			        "Unable to shutdown PhysicsFS: " << PHYSFS_getErrorByCode((PHYSFS_getLastErrorCode())));
294
		}
295
296
	}

297
298
	void Asset_manager::reload()
	{
299
		_reload_dispatchers();
300

301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
		// The container lock must not be held during reload, because the reload of an asset calls
		//   third-party code that might call into the asset_manager.
		// So we first collect all relevant containers and then iterate over that list.
		auto containers = std::vector<detail::Asset_container_base*>();
		{
			auto lock = std::scoped_lock{_containers_mutex};
			containers.reserve(_containers.size());

			for(auto& container : _containers) {
				containers.emplace_back(container.second.get());
			}
		}

		for(auto& c : containers) {
			c->reload();
316
317
		}
	}
318
319
	void Asset_manager::shrink_to_fit() noexcept
	{
320
		auto lock = std::scoped_lock{_containers_mutex};
321

322
323
324
		for(auto& container : _containers) {
			container.second->shrink_to_fit();
		}
325
326
	}

327
328
	auto Asset_manager::exists(const AID& id) const noexcept -> bool
	{
329
330
		return resolve(id).process(false, [](auto&& path) { return exists_file(path); });
	}
331
332
	auto Asset_manager::try_delete(const AID& id) -> bool
	{
333
		return resolve(id).process(true, [](auto&& path) { return PHYSFS_delete(path.c_str()) == 0; });
334
	}
335

336
337
	auto Asset_manager::open(const AID& id) -> util::maybe<istream>
	{
338
339
340
341
342
		auto path = resolve(id);

		if(path.is_some() && exists_file(path.get_or_throw()))
			return _open(id, path.get_or_throw());
		else
343
			return util::nothing;
344
	}
345
346
	auto Asset_manager::open_rw(const AID& id) -> ostream
	{
347
		auto path = resolve(id, false);
348
349
350
351
352
		if(path.is_nothing()) {
			path = resolve(AID{id.type(), ""}).process(id.str(), [&](auto&& prefix) {
				return append_file(prefix, id.str());
			});
		};
353

354
355
356
357
		if(exists_file(path.get_or_throw()))
			PHYSFS_delete(path.get_or_throw().c_str());

		return _open_rw(id, path.get_or_throw());
358
359
	}

360
361
	auto Asset_manager::list(Asset_type type) -> std::vector<AID>
	{
362
363
		auto lock = std::shared_lock{_dispatchers_mutex};

364
365
		std::vector<AID> res;

366
		for(auto& d : _dispatchers) {
367
			if(d.first.type() == type && d.first.name().size() > 0)
368
369
370
				res.emplace_back(d.first);
		}

371
372
		util::find_maybe(_general_dispatchers, type).process([&](auto& entry) {
			for(auto&& f : list_files(entry.base_dir, "", ""))
373
374
375
376
377
378
379
380
				res.emplace_back(type, f);
		});

		sort(res.begin(), res.end());
		res.erase(std::unique(res.begin(), res.end()), res.end());

		return res;
	}
381
382
	auto Asset_manager::last_modified(const AID& id) const noexcept -> util::maybe<std::int64_t>
	{
383
		using namespace std::literals;
384

385
		return resolve(id).process([&](auto& path) { return _last_modified(path); });
386
387
	}

388
	auto Asset_manager::resolve(const AID& id, bool only_preexisting) const noexcept
389
390
	        -> util::maybe<std::string>
	{
391
		auto lock = std::shared_lock{_dispatchers_mutex};
392

393
		auto res = _dispatchers.find(id);
394

395
		if(res != _dispatchers.end() && (exists_file(res->second) || !only_preexisting))
396
			return res->second;
397

398
399
		else if(exists_file(id.name()))
			return id.name();
400
401


402
403
		if(auto try2 = _resolve_unkown(id, only_preexisting); try2.is_some()) {
			return std::move(try2.get_or_throw());
404
405
406
407
		}

		if(!only_preexisting) {
			return id.name();
408
409
		}

410
		LOG(plog::warning) << "Couldn't resove AID '" << id.str()
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
		                   << "'. Dispatcher: " << (res != _dispatchers.end() ? res->second : "!NO-MATCH");

		return util::nothing;
	}
	auto Asset_manager::_resolve_unkown(const AID& id, bool only_preexisting) const
	        -> util::maybe<std::string>
	{
		auto lock = std::shared_lock{_dispatchers_mutex};

		auto dir = _general_dispatchers.find(id.type());

		if(dir == _general_dispatchers.end())
			return util::nothing;

		auto path = append_file(dir->second.base_dir, id.name());
		if(exists_file(path))
			return std::move(path);

		path = path + dir->second.default_extension;
		if(exists_file(path))
			return std::move(path);

		if(!only_preexisting) {
			PHYSFS_mkdir(dir->second.base_dir.c_str());
			return std::move(path);
		}
437

438
		return util::nothing;
439
	}
440
441
	auto Asset_manager::resolve_reverse(std::string_view path) -> util::maybe<AID>
	{
442
		auto lock = std::shared_lock{_dispatchers_mutex};
443

444
445
446
		for(auto& e : _dispatchers) {
			if(e.second == path)
				return e.first;
447
448
		}

449
		return util::nothing;
450
451
	}

452
	void Asset_manager::_post_write() {}
453

454
455
	void Asset_manager::_reload_dispatchers()
	{
456
		auto lock = std::unique_lock{_dispatchers_mutex};
457

458
		_dispatchers.clear();
459

460
		for(auto&& df : list_files("", "assets", ".map")) {
461
			LOG(plog::info) << "Added asset mapping: " << df;
462
463
464
465
466
			auto in = _open({}, df);
			for(auto&& l : in.lines()) {
				auto        kvp  = util::split(l, "=");
				std::string path = util::trim_copy(kvp.second);
				if(!path.empty()) {
467
					auto aid = AID{kvp.first};
468
					LOG(plog::debug) << "    " << AID{kvp.first}.str() << " = " << path;
469
470
471
472
473
474

					if(aid.name().empty()) {
						auto [dir, ext] = util::split_on_last(path, "*");
						_general_dispatchers.emplace(aid.type(), General_Disptacher{dir, ext});
					} else
						_dispatchers.emplace(std::move(aid), std::move(path));
475
476
477
478
479
				}
			}
		}
	}

480
481
	auto Asset_manager::_last_modified(const std::string& path) const -> int64_t
	{
482
		auto stat = PHYSFS_Stat{};
483
484
		if(!PHYSFS_stat(path.c_str(), &stat))
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()));
485

486
		return stat.modtime;
487
	}
488
489
	auto Asset_manager::_open(const asset::AID& id, const std::string& path) -> istream
	{
490
		return {id, *this, path};
491
	}
492
493
	auto Asset_manager::_open_rw(const asset::AID& id, const std::string& path) -> ostream
	{
494
		return {id, *this, path};
495
496
	}

497
} // namespace mirrage::asset