asset_manager.cpp 12.5 KB
Newer Older
1
2
#include <mirrage/asset/asset_manager.hpp>

3
4
#include <mirrage/asset/error.hpp>

5
#include <mirrage/utils/log.hpp>
6
#include <mirrage/utils/template_utils.hpp>
7
8
9
10

#include <physfs.h>

#include <cstdio>
11
#include <cstring>
12

Florian Oetke's avatar
Florian Oetke committed
13
#ifdef _WIN32
14
15
#include <direct.h>
#include <windows.h>
16
#else
17
18
#include <sys/stat.h>
#include <unistd.h>
19
20
21
#endif

#ifdef EMSCRIPTEN
22
#include <emscripten.h>
23
24
#endif

25
using namespace mirrage::util;
26
using namespace std::string_literals;
27
28
29
30
31

namespace {

	std::string append_file(const std::string& folder, const std::string file) {
		if(ends_with(folder, "/") || starts_with(file, "/"))
32
			return folder + file;
33
		else
34
			return folder + "/" + file;
35
36
	}
	void create_dir(const std::string& dir) {
Florian Oetke's avatar
Florian Oetke committed
37
#ifdef _WIN32
38
39
40
41
42
43
44
45
		CreateDirectory(dir.c_str(), NULL);
#else
		mkdir(dir.c_str(), 0777);
#endif
	}

	auto last_of(const std::string& str, char c) {
		auto idx = str.find_last_of(c);
46
		return idx != std::string::npos ? mirrage::util::just(idx + 1) : mirrage::util::nothing;
47
48
49
	}

	auto split_path(const std::string& path) {
50
		auto filename_delim_end = last_of(path, '/').get_or(last_of(path, '\\').get_or(0));
51

52
		return std::make_tuple(path.substr(0, filename_delim_end - 1),
53
54
55
56
		                       path.substr(filename_delim_end, std::string::npos));
	}


57
58
59
	std::vector<std::string> list_files(const std::string& dir,
	                                    const std::string& prefix,
	                                    const std::string& suffix) noexcept {
60
61
		std::vector<std::string> res;

62
		char** rc = PHYSFS_enumerateFiles(dir.c_str());
63

64
		for(char** i = rc; *i != nullptr; i++) {
65
66
			std::string str(*i);

67
68
			if((prefix.length() == 0 || str.find(prefix) == 0)
			   && (suffix.length() == 0 || str.find(suffix) == str.length() - suffix.length()))
69
70
71
72
73
74
75
76
				res.emplace_back(std::move(str));
		}

		PHYSFS_freeList(rc);

		return res;
	}
	std::vector<std::string> list_wildcard_files(const std::string& wildcard_path) {
77
		auto   spr  = split_path(wildcard_path);
78
79
80
		auto&& path = std::get<0>(spr);
		auto&& file = std::get<1>(spr);

81
		auto wildcard = last_of(file, '*').get_or(file.length());
82
		if(wildcard != (file.find_first_of('*') + 1)) {
83
			MIRRAGE_WARN("More than one wildcard ist currently not supported. Found in: " << wildcard_path);
84
85
		}

86
87
		auto prefix = file.substr(0, wildcard - 1);
		auto suffix = wildcard < file.length() ? file.substr(0, wildcard - 1) : std::string();
88
89
90
91
92
93
94
95
96
97

		auto files = list_files(path, prefix, suffix);
		for(auto& file : files) {
			file = path + "/" + file;
		}

		return files;
	}

	bool exists_file(const std::string path) {
98
		if(!PHYSFS_exists(path.c_str()))
99
100
101
			return false;

		auto stat = PHYSFS_Stat{};
102
		if(!PHYSFS_stat(path.c_str(), &stat))
103
104
105
			return false;

		return stat.filetype == PHYSFS_FILETYPE_REGULAR;
106
107
	}
	bool exists_dir(const std::string path) {
108
		if(!PHYSFS_exists(path.c_str()))
109
110
111
			return false;

		auto stat = PHYSFS_Stat{};
112
		if(!PHYSFS_stat(path.c_str(), &stat))
113
114
115
			return false;

		return stat.filetype == PHYSFS_FILETYPE_DIRECTORY;
116
117
	}

118
	template <typename Stream>
119
120
	void print_dir_recursiv(const std::string& dir, uint8_t depth, Stream& stream) {
		std::string p;
121
122
		for(uint8_t i = 0; i < depth; i++)
			p += "  ";
123

124
		stream << p << dir << "\n";
125
126
		depth++;
		for(auto&& f : list_files(dir, "", "")) {
127
128
			if(depth >= 5)
				stream << p << "  " << f << "\n";
129
130
131
132
133
			else
				print_dir_recursiv(f, depth, stream);
		}
	}

134
	constexpr auto default_source = {std::make_tuple("assets", false), std::make_tuple("assets.zip", true)};
135
} // namespace
136

137
namespace mirrage::asset {
138
139
140
141

	std::string pwd() {
		char cCurrentPath[FILENAME_MAX];

142
143
144
#ifdef WINDOWS
		_getcwd(cCurrentPath, sizeof(cCurrentPath));
#else
145
146
147
		if(getcwd(cCurrentPath, sizeof(cCurrentPath)) == nullptr) {
			MIRRAGE_FAIL("getcwd with max length " << FILENAME_MAX << " failed with error code " << errno);
		}
148
#endif
149
150
151
152
153
154
155
156

		return cCurrentPath;
	}


#ifdef EMSCRIPTEN
	static bool initial_sync_done = false;

157
	extern "C" void EMSCRIPTEN_KEEPALIVE post_sync_handler() { initial_sync_done = true; }
158
159

	void setup_storage() {
160
161
162
		EM_ASM(FS.mkdir('/persistent_data'); FS.mount(IDBFS, {}, '/persistent_data');

		       Module.syncdone = 0;
163

164
165
166
167
168
169
170
171
172
173
174
		       //populate persistent_data directory with existing persistent source data
		       //stored with Indexed Db
		       //first parameter = "true" mean synchronize from Indexed Db to
		       //Emscripten file system,
		       // "false" mean synchronize from Emscripten file system to Indexed Db
		       //second parameter = function called when data are synchronized
		       FS.syncfs(true, function(err) {
			       //assert(!err);
			       Module.print("end file sync..");
			       Module.syncdone = 1;
			       ccall('post_sync_handler', 'v');
175
		       }););
176
	}
177
178

	bool storage_ready() { return initial_sync_done; }
179
180
#else
	void setup_storage() {}
181
	bool storage_ready() { return true; }
182
183
184
185

#endif


186
187
188
	Asset_manager::Asset_manager(const std::string& exe_name,
	                             const std::string& org_name,
	                             const std::string& app_name) {
189
		if(!PHYSFS_init(exe_name.empty() ? nullptr : exe_name.c_str()))
190
191
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to initalize PhysicsFS.");
192

193
		auto write_dir = PHYSFS_getPrefDir(org_name.c_str(), app_name.c_str());
194
195

#ifdef EMSCRIPTEN
196
		MIRRAGE_INVARIANT(storage_ready(), "Storage is not ready");
197
		write_dir = "/persistent_data";
198
199
#endif

200
201
202
203
204
		if(!PHYSFS_mount(PHYSFS_getBaseDir(), nullptr, 1)
		   || !PHYSFS_mount(append_file(PHYSFS_getBaseDir(), "..").c_str(), nullptr, 1)
		   || !PHYSFS_mount(pwd().c_str(), nullptr, 1))
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to setup default search path.");
205
206
207


		if(exists_dir("write_dir")) {
208
			write_dir = "write_dir";
209
210
		}

211
212
		create_dir(write_dir);

213
		MIRRAGE_INFO("Write dir: " << write_dir);
214

215
216
217
		if(!PHYSFS_mount(write_dir, nullptr, 0))
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to construct search path.");
218

219
220
221
		if(!PHYSFS_setWriteDir(write_dir))
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
			                        "Unable to set write-dir: "s + write_dir);
222
223
224


		auto add_source = [](const char* path) {
225
226
227
			if(!PHYSFS_mount(path, nullptr, 1))
				throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
				                        "Error adding custom archive: "s + path);
228
229
		};

230
		if(!exists_file("archives.lst")) {
231
232
233
			bool lost = true;
			for(auto& s : default_source) {
				const char* path;
234
				bool        file;
235
236
237
238
239
240
241
242
243
244

				std::tie(path, file) = s;

				if(file ? exists_file(path) : exists_dir(path)) {
					add_source(path);
					lost = false;
				}
			}

			if(lost) {
245
				auto& log = util::error(__func__, __FILE__, __LINE__);
246
				log << "No archives.lst found. printing search-path...\n";
247
				print_dir_recursiv("/", 0, log);
248
				log << std::endl;
249

250
251
				throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()),
				                        "No archives.lst found.");
252
253

			} else {
254
				MIRRAGE_INFO("No archives.lst found. Using defaults.");
255
256
257
			}

		} else {
258
259
			auto in = _open("cfg:archives.lst"_aid, "archives.lst");

260
			// load other archives
261
262
263
			for(auto&& l : in.lines()) {
				if(l.find_last_of('*') != std::string::npos) {
					for(auto& file : list_wildcard_files(l)) {
264
						MIRRAGE_INFO("Added FS directory: " << file);
265
						add_source(file.c_str());
266
					}
267
					continue;
268
				}
269
270
				add_source(l.c_str());
			}
271
272
273
274
275
276
		}

		_reload_dispatchers();
	}

	Asset_manager::~Asset_manager() {
277
278
		_containers.clear();
		if(!PHYSFS_deinit()) {
279
280
			MIRRAGE_FAIL(
			        "Unable to shutdown PhysicsFS: " << PHYSFS_getErrorByCode((PHYSFS_getLastErrorCode())));
281
		}
282
283
	}

284
285
	void Asset_manager::reload() {
		_reload_dispatchers();
286

287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
		// The container lock must not be held during reload, because the reload of an asset calls
		//   third-party code that might call into the asset_manager.
		// So we first collect all relevant containers and then iterate over that list.
		auto containers = std::vector<detail::Asset_container_base*>();
		{
			auto lock = std::scoped_lock{_containers_mutex};
			containers.reserve(_containers.size());

			for(auto& container : _containers) {
				containers.emplace_back(container.second.get());
			}
		}

		for(auto& c : containers) {
			c->reload();
302
303
		}
	}
304
305
	void Asset_manager::shrink_to_fit() noexcept {
		auto lock = std::scoped_lock{_containers_mutex};
306

307
308
309
		for(auto& container : _containers) {
			container.second->shrink_to_fit();
		}
310
311
	}

312
313
314
315
	auto Asset_manager::exists(const AID& id) const noexcept -> bool {
		return resolve(id).process(false, [](auto&& path) { return exists_file(path); });
	}
	auto Asset_manager::try_delete(const AID& id) -> bool {
316
		return resolve(id).process(true, [](auto&& path) { return PHYSFS_delete(path.c_str()) == 0; });
317
	}
318

319
320
321
322
323
324
	auto Asset_manager::open(const AID& id) -> util::maybe<istream> {
		auto path = resolve(id);

		if(path.is_some() && exists_file(path.get_or_throw()))
			return _open(id, path.get_or_throw());
		else
325
			return util::nothing;
326
327
328
329
330
331
332
333
	}
	auto Asset_manager::open_rw(const AID& id) -> ostream {
		auto path = resolve(id);
		if(path.is_nothing()) {
			path = resolve(AID{id.type(), ""}).process(id.str(), [&](auto&& prefix) {
				return append_file(prefix, id.str());
			});
		};
334

335
336
337
338
		if(exists_file(path.get_or_throw()))
			PHYSFS_delete(path.get_or_throw().c_str());

		return _open_rw(id, path.get_or_throw());
339
340
	}

341
342
343
	auto Asset_manager::list(Asset_type type) -> std::vector<AID> {
		auto lock = std::shared_lock{_dispatchers_mutex};

344
345
		std::vector<AID> res;

346
		for(auto& d : _dispatchers) {
347
			if(d.first.type() == type && d.first.name().size() > 0)
348
349
350
				res.emplace_back(d.first);
		}

351
		_base_dir(type).process([&](const std::string& dir) {
352
353
354
355
356
357
358
359
360
			for(auto&& f : list_files(dir, "", ""))
				res.emplace_back(type, f);
		});

		sort(res.begin(), res.end());
		res.erase(std::unique(res.begin(), res.end()), res.end());

		return res;
	}
361
362
	auto Asset_manager::last_modified(const AID& id) const noexcept -> util::maybe<std::int64_t> {
		using namespace std::literals;
363

364
		return resolve(id).process([&](auto& path) { return _last_modified(path); });
365
366
	}

367
368
	auto Asset_manager::resolve(const AID& id, bool only_preexisting) const noexcept
	        -> util::maybe<std::string> {
369
		auto lock = std::shared_lock{_dispatchers_mutex};
370

371
		auto res = _dispatchers.find(id);
372

373
		if(res != _dispatchers.end() && (exists_file(res->second) || !only_preexisting))
374
			return res->second;
375

376
377
		else if(exists_file(id.name()))
			return id.name();
378
379
380
381
382
383
384


		auto baseDir = _base_dir(id.type());

		if(baseDir.is_some()) {
			auto path = append_file(baseDir.get_or_throw(), id.name());
			if(exists_file(path))
385
				return std::move(path);
386
387
388
389
390
391
392
393
394

			else if(!only_preexisting) {
				PHYSFS_mkdir(baseDir.get_or_throw().c_str());
				return std::move(path);
			}
		}

		if(!only_preexisting) {
			return id.name();
395
396
		}

397
		return util::nothing;
398
	}
399
400
	auto Asset_manager::resolve_reverse(std::string_view path) -> util::maybe<AID> {
		auto lock = std::shared_lock{_dispatchers_mutex};
401

402
403
404
		for(auto& e : _dispatchers) {
			if(e.second == path)
				return e.first;
405
406
		}

407
		return util::nothing;
408
409
	}

410
411
412
413
414
415
416
417
	void Asset_manager::_post_write() {
#ifdef EMSCRIPTEN
		//persist Emscripten current data to Indexed Db
		EM_ASM(FS.syncfs(false,
		                 function(err){
		                         //assert(!err);
		                 }););
#endif
418
419
	}

420
421
	auto Asset_manager::_base_dir(Asset_type type) const -> util::maybe<std::string> {
		auto lock = std::shared_lock{_dispatchers_mutex};
422

423
		auto dir = _dispatchers.find(AID{type, ""}); // search for prefix-entry
424

425
		if(dir == _dispatchers.end())
426
427
			return util::nothing;

428
429
		std::string bdir = dir->second;
		return bdir;
430
431
	}

432
433
	void Asset_manager::_reload_dispatchers() {
		auto lock = std::unique_lock{_dispatchers_mutex};
434

435
		_dispatchers.clear();
436

437
		for(auto&& df : list_files("", "assets", ".map")) {
438
			MIRRAGE_INFO("Added asset mapping: " << df);
439
440
441
442
443
444
			auto in = _open({}, df);
			for(auto&& l : in.lines()) {
				auto        kvp  = util::split(l, "=");
				std::string path = util::trim_copy(kvp.second);
				if(!path.empty()) {
					_dispatchers.emplace(AID{kvp.first}, std::move(path));
445
446
447
448
449
				}
			}
		}
	}

450
451
	auto Asset_manager::_last_modified(const std::string& path) const -> int64_t {
		auto stat = PHYSFS_Stat{};
452
453
		if(!PHYSFS_stat(path.c_str(), &stat))
			throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()));
454

455
		return stat.modtime;
456
	}
457
458
	auto Asset_manager::_open(const asset::AID& id, const std::string& path) -> istream {
		return {id, *this, path};
459
	}
460
461
	auto Asset_manager::_open_rw(const asset::AID& id, const std::string& path) -> ostream {
		return {id, *this, path};
462
463
	}

464
} // namespace mirrage::asset