asset_manager.cpp 15.6 KB
Newer Older
1
2
3
#include <mirrage/asset/asset_manager.hpp>

#include <mirrage/utils/log.hpp>
4
#include <mirrage/utils/template_utils.hpp>
5
6
7
8

#include <physfs.h>

#include <cstdio>
9
#include <cstring>
10

Florian Oetke's avatar
Florian Oetke committed
11
#ifdef _WIN32
12
13
#include <direct.h>
#include <windows.h>
14
#else
15
16
#include <sys/stat.h>
#include <unistd.h>
17
18
19
#endif

#ifdef EMSCRIPTEN
20
#include <emscripten.h>
21
22
#endif

23
using namespace mirrage::util;
24
25
26
27
28

namespace {

	std::string append_file(const std::string& folder, const std::string file) {
		if(ends_with(folder, "/") || starts_with(file, "/"))
29
			return folder + file;
30
		else
31
			return folder + "/" + file;
32
33
	}
	void create_dir(const std::string& dir) {
Florian Oetke's avatar
Florian Oetke committed
34
#ifdef _WIN32
35
36
37
38
39
40
41
42
		CreateDirectory(dir.c_str(), NULL);
#else
		mkdir(dir.c_str(), 0777);
#endif
	}

	auto last_of(const std::string& str, char c) {
		auto idx = str.find_last_of(c);
43
		return idx != std::string::npos ? mirrage::util::just(idx + 1) : mirrage::util::nothing;
44
45
46
47
48
	}

	auto split_path(const std::string& path) {
		auto filename_delim_end = last_of(path, '/').get_or_other(last_of(path, '\\').get_or_other(0));

49
		return std::make_tuple(path.substr(0, filename_delim_end - 1),
50
51
52
53
		                       path.substr(filename_delim_end, std::string::npos));
	}


54
55
56
	std::vector<std::string> list_files(const std::string& dir,
	                                    const std::string& prefix,
	                                    const std::string& suffix) noexcept {
57
58
		std::vector<std::string> res;

59
		char** rc = PHYSFS_enumerateFiles(dir.c_str());
60

61
		for(char** i = rc; *i != nullptr; i++) {
62
63
			std::string str(*i);

64
65
			if((prefix.length() == 0 || str.find(prefix) == 0)
			   && (suffix.length() == 0 || str.find(suffix) == str.length() - suffix.length()))
66
67
68
69
70
71
72
73
				res.emplace_back(std::move(str));
		}

		PHYSFS_freeList(rc);

		return res;
	}
	std::vector<std::string> list_wildcard_files(const std::string& wildcard_path) {
74
		auto   spr  = split_path(wildcard_path);
75
76
77
78
		auto&& path = std::get<0>(spr);
		auto&& file = std::get<1>(spr);

		auto wildcard = last_of(file, '*').get_or_other(file.length());
79
		if(wildcard != (file.find_first_of('*') + 1)) {
80
81
82
			WARN("More than one wildcard ist currently not supported. Found in: " << wildcard_path);
		}

83
84
		auto prefix = file.substr(0, wildcard - 1);
		auto suffix = wildcard < file.length() ? file.substr(0, wildcard - 1) : std::string();
85
86
87
88
89
90
91
92
93
94

		auto files = list_files(path, prefix, suffix);
		for(auto& file : files) {
			file = path + "/" + file;
		}

		return files;
	}

	bool exists_file(const std::string path) {
95
		return PHYSFS_exists(path.c_str()) != 0 && PHYSFS_isDirectory(path.c_str()) == 0;
96
97
	}
	bool exists_dir(const std::string path) {
98
		return PHYSFS_exists(path.c_str()) != 0 && PHYSFS_isDirectory(path.c_str()) != 0;
99
100
	}

101
	template <typename Stream>
102
103
	void print_dir_recursiv(const std::string& dir, uint8_t depth, Stream& stream) {
		std::string p;
104
105
		for(uint8_t i = 0; i < depth; i++)
			p += "  ";
106

107
		stream << p << dir << "\n";
108
109
		depth++;
		for(auto&& f : list_files(dir, "", "")) {
110
111
			if(depth >= 5)
				stream << p << "  " << f << "\n";
112
113
114
115
116
117
118
119
			else
				print_dir_recursiv(f, depth, stream);
		}
	}

	constexpr auto default_source = {std::make_tuple("assets", false), std::make_tuple("assets.zip", true)};
}

120
namespace mirrage::asset {
121
122
123
124

	std::string pwd() {
		char cCurrentPath[FILENAME_MAX];

125
126
127
128
129
#ifdef WINDOWS
		_getcwd(cCurrentPath, sizeof(cCurrentPath));
#else
		getcwd(cCurrentPath, sizeof(cCurrentPath));
#endif
130
131
132
133
134
135
136
137

		return cCurrentPath;
	}


#ifdef EMSCRIPTEN
	static bool initial_sync_done = false;

138
	extern "C" void EMSCRIPTEN_KEEPALIVE post_sync_handler() { initial_sync_done = true; }
139
140

	void setup_storage() {
141
142
143
		EM_ASM(FS.mkdir('/persistent_data'); FS.mount(IDBFS, {}, '/persistent_data');

		       Module.syncdone = 0;
144

145
146
147
148
149
150
151
152
153
154
155
156
		       //populate persistent_data directory with existing persistent source data
		       //stored with Indexed Db
		       //first parameter = "true" mean synchronize from Indexed Db to
		       //Emscripten file system,
		       // "false" mean synchronize from Emscripten file system to Indexed Db
		       //second parameter = function called when data are synchronized
		       FS.syncfs(true, function(err) {
			       //assert(!err);
			       Module.print("end file sync..");
			       Module.syncdone = 1;
			       ccall('post_sync_handler', 'v');
			   }););
157
	}
158
159

	bool storage_ready() { return initial_sync_done; }
160
161
#else
	void setup_storage() {}
162
	bool storage_ready() { return true; }
163
164
165
166

#endif

	static Asset_manager* current_instance = nullptr;
167
168
	auto                  get_asset_manager() -> Asset_manager& {
		INVARIANT(current_instance != nullptr, "Asset_manager has not been initialized!");
169
170
171
172
173
		return *current_instance;
	}

	Asset_manager::Asset_manager(const std::string& exe_name, const std::string& app_name) {
		if(!PHYSFS_init(exe_name.empty() ? nullptr : exe_name.c_str()))
174
			FAIL("PhysFS-Init failed for \"" << exe_name << "\": " << PHYSFS_getLastError());
175
176
177

		// TODO: Windows savegames should be stored in FOLDERID_SavedGames, but the API and conventions are a pain in the ass
		std::string write_dir_parent = append_file(PHYSFS_getUserDir(),
Florian Oetke's avatar
Florian Oetke committed
178
#ifdef _WIN32
179
		                                           "Documents/My Games"
180
#else
181
		                                           ".config"
182
#endif
183
		                                           );
184
185
186
187
188
189

#ifdef EMSCRIPTEN
		INVARIANT(storage_ready(), "Storage is not ready");
		write_dir_parent = "/persistent_data";
#endif

190
191
192
193
194
		if(!PHYSFS_addToSearchPath(PHYSFS_getBaseDir(), 1)
		   || !PHYSFS_addToSearchPath(append_file(PHYSFS_getBaseDir(), "..").c_str(), 1)
		   || !PHYSFS_addToSearchPath(pwd().c_str(), 1))
			FAIL("Unable to construct search path: " << PHYSFS_getLastError());

195
		// add optional search path
196
197
198
		PHYSFS_addToSearchPath(
		        append_file(append_file(append_file(PHYSFS_getBaseDir(), ".."), app_name), "assets").c_str(),
		        1);
199
200
201
202
203
204


		if(exists_dir("write_dir")) {
			write_dir_parent = "write_dir";
		}

205
206
207
208
209
		create_dir(write_dir_parent);

		std::string write_dir = append_file(write_dir_parent, app_name);
		create_dir(write_dir);

210
		INFO("Write dir: " << write_dir);
211

212
		if(!PHYSFS_addToSearchPath(write_dir.c_str(), 0))
213
			FAIL("Unable to construct search path: " << PHYSFS_getLastError());
214
215

		if(!PHYSFS_setWriteDir(write_dir.c_str()))
216
			FAIL("Unable to set write-dir to \"" << write_dir << "\": " << PHYSFS_getLastError());
217
218
219
220


		auto add_source = [](const char* path) {
			if(!PHYSFS_addToSearchPath(path, 1))
221
				WARN("Error adding custom archive \"" << path << "\": " << PHYSFS_getLastError());
222
223
224
225
226
227
228
		};

		auto archive_file = _open("archives.lst");
		if(!archive_file) {
			bool lost = true;
			for(auto& s : default_source) {
				const char* path;
229
				bool        file;
230
231
232
233
234
235
236
237
238
239

				std::tie(path, file) = s;

				if(file ? exists_file(path) : exists_dir(path)) {
					add_source(path);
					lost = false;
				}
			}

			if(lost) {
240
241
				auto& log = util::fail(__func__, __FILE__, __LINE__);
				log << "No archives.lst found. printing search-path...\n";
242
243
				print_dir_recursiv("/", 0, log);

244
				log << std::endl; // crash with error
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281

			} else {
				INFO("No archives.lst found. Using defaults.");
			}

		} else {
			// load other archives
			archive_file.process([&](istream& in) {
				for(auto&& l : in.lines()) {
					if(l.find_last_of('*') != std::string::npos) {
						for(auto& file : list_wildcard_files(l)) {
							add_source(file.c_str());
						}
						continue;
					}
					add_source(l.c_str());
				}
			});
		}

		_reload_dispatchers();

		current_instance = this;
	}

	Asset_manager::~Asset_manager() {
		current_instance = nullptr;
		_assets.clear();
		PHYSFS_deinit();
	}

	void Asset_manager::_reload_dispatchers() {
		_dispatcher.clear();

		for(auto&& df : list_files("", "assets", ".map")) {
			_open(df).process([this](istream& in) {
				for(auto&& l : in.lines()) {
282
					auto        kvp  = util::split(l, "=");
283
284
285
286
287
288
289
290
291
292
293
294
					std::string path = util::trim_copy(kvp.second);
					if(!path.empty()) {
						_dispatcher.emplace(AID{kvp.first}, std::move(path));
					}
				}
			});
		}
	}

	void Asset_manager::_post_write() {
#ifdef EMSCRIPTEN
		//persist Emscripten current data to Indexed Db
295
296
297
298
		EM_ASM(FS.syncfs(false,
		                 function(err){
		                         //assert(!err);
		                 }););
299
300
301
302
#endif
		reload();
	}

303
	util::maybe<std::string> Asset_manager::_base_dir(Asset_type type) const {
304
305
		auto dir = _dispatcher.find(AID{type, ""}); // search for prefix-entry

306
		if(dir == _dispatcher.end())
307
308
309
310
311
312
313
314
315
			return util::nothing;

		std::string bdir = dir->second;
		return bdir;
	}

	std::vector<AID> Asset_manager::list(Asset_type type) {
		std::vector<AID> res;

316
317
		for(auto& d : _dispatcher) {
			if(d.first.type() == type && d.first.name().size() > 0)
318
319
320
				res.emplace_back(d.first);
		}

321
		_base_dir(type).process([&](const std::string& dir) {
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
			for(auto&& f : list_files(dir, "", ""))
				res.emplace_back(type, f);
		});

		sort(res.begin(), res.end());
		res.erase(std::unique(res.begin(), res.end()), res.end());

		return res;
	}

	util::maybe<istream> Asset_manager::_open(const std::string& path) {
		return _open(path, AID{"gen"_strid, path});
	}
	util::maybe<istream> Asset_manager::_open(const std::string& path, const AID& aid) {
		return exists_file(path) ? util::just(istream{aid, *this, path}) : util::nothing;
	}

	Asset_manager::Asset::Asset(std::shared_ptr<void> data, Reloader reloader, int64_t last_modified)
340
	  : data(data), reloader(reloader), last_modified(last_modified) {}
341

342
343
344
345
	void Asset_manager::_add_asset(const AID&            id,
	                               const std::string&    path,
	                               Reloader              reloader,
	                               std::shared_ptr<void> asset) {
346
347
348
		_assets.emplace(id, Asset{asset, reloader, PHYSFS_getLastModTime(path.c_str())});
	}

349
	auto Asset_manager::_locate(const AID& id, bool warn) const -> std::tuple<Location_type, std::string> {
350
351
		auto res = _dispatcher.find(id);

352
		if(res != _dispatcher.end()) {
353
354
355
356
357
			if(exists_file(res->second))
				return std::make_tuple(Location_type::file, res->second);
			else if(util::contains(res->second, ":"))
				return std::make_tuple(Location_type::indirection, res->second);
			else if(warn)
358
				INFO("Asset not found in configured place: " << res->second);
359
360
361
362
363
364
365
366
367
368
369
370
		}

		if(exists_file(id.name()))
			return std::make_tuple(Location_type::file, id.name());

		auto baseDir = _base_dir(id.type());

		if(baseDir.is_some()) {
			auto path = append_file(baseDir.get_or_throw(), id.name());
			if(exists_file(path))
				return std::make_tuple(Location_type::file, std::move(path));
			else if(warn)
371
				DEBUG("asset " << id.str() << " not found in " << path);
372
373
374
375
376
377
378
379
380
		}

		return std::make_tuple(Location_type::none, std::string());
	}

	ostream Asset_manager::_create(const AID& id) {
		std::string path;

		auto path_res = _dispatcher.find(id);
381
		if(path_res != _dispatcher.end())
382
383
384
385
386
			path = path_res->second;

		else {
			auto res = _dispatcher.find(AID{id.type(), ""}); // search for prefix-entry

387
			if(res != _dispatcher.end()) {
388
389
390
391
392
393
394
395
396
397
398
399
400
				PHYSFS_mkdir(res->second.c_str());
				path = append_file(res->second, id.name());
			} else {
				path = id.name();
			}
		}

		if(exists_file(path))
			PHYSFS_delete(path.c_str());

		return {id, *this, path};
	}

401
402
	auto Asset_manager::physical_location(const AID& id, bool warn) const noexcept
	        -> util::maybe<std::string> {
403
404
405
		using namespace std::literals;

		Location_type type;
406
		std::string   location;
407
408
		std::tie(type, location) = _locate(id, warn);

409
		if(type != Location_type::file)
410
411
412
413
414
415
416
417
418
419
			return util::nothing;

		auto dir = PHYSFS_getRealDir(location.c_str());
		if(!dir)
			return util::nothing;

		auto file = dir + "/"s + location;
		return exists_file(file) ? util::just(std::move(file)) : util::nothing;
	}

420
	auto Asset_manager::last_modified(const AID& id) const noexcept -> util::maybe<std::int64_t> {
421
422
423
		using namespace std::literals;

		Location_type type;
424
		std::string   location;
425
426
		std::tie(type, location) = _locate(id, false);

427
		if(type != Location_type::file)
428
429
430
431
432
433
434
435
436
			return util::nothing;

		return PHYSFS_getLastModTime(location.c_str());
	}

	auto Asset_manager::try_delete(const AID& id) -> bool {
		using namespace std::literals;

		Location_type type;
437
		std::string   location;
438
439
		std::tie(type, location) = _locate(id, false);

440
		if(type != Location_type::file)
441
442
443
444
445
446
447
448
449
			return false;

		return PHYSFS_delete(location.c_str());
	}

	void Asset_manager::reload() {
		_reload_dispatchers();
		for(auto& a : _assets) {
			Location_type type;
450
			std::string   location;
451
452
			std::tie(type, location) = _locate(a.first);

453
			if(type == Location_type::file) {
454
				auto last_mod = PHYSFS_getLastModTime(location.c_str());
455
456
457
				if(last_mod != -1 && last_mod > a.second.last_modified) {
					_open(location, a.first).process([&](istream& in) {
						DEBUG("Reload: " << a.first.str());
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
						try {
							a.second.reloader(a.second.data.get(), std::move(in));

						} catch(Loading_failed&) {}

						a.second.last_modified = last_mod;
					});
				}
			}
		}

		for(auto& w : _watchlist) {
			_check_watch_entry(w);
		}
	}
	void Asset_manager::_force_reload(const AID& aid) {
		auto iter = _assets.find(aid);
475
		if(iter == _assets.end())
476
477
478
			return;

		Location_type type;
479
		std::string   location;
480
481
		std::tie(type, location) = _locate(aid);

482
483
		if(type == Location_type::file) {
			_open(location, aid).process([&](istream& in) {
484
485
486
				try {
					iter->second.reloader(iter->second.data.get(), std::move(in));
					for(auto& w : _watchlist) {
487
						if(w.aid == aid) {
488
489
490
491
492
493
494
495
496
497
							w.on_mod(aid);
							w.last_modified = PHYSFS_getLastModTime(location.c_str());
						}
					}

				} catch(Loading_failed&) {}
			});
		}
	}

498
499
	void Asset_manager::shrink_to_fit() noexcept {
		util::erase_if(_assets, [](const auto& v) { return v.second.data.use_count() <= 1; });
500
501
	}

502
	bool Asset_manager::exists(const AID& id) const noexcept {
503
		Location_type type;
504
		std::string   location;
505
506
507
		std::tie(type, location) = _locate(id);

		switch(type) {
508
			case Location_type::none: return false;
509

510
			case Location_type::file: return exists_file(location);
511

512
			case Location_type::indirection: return true;
513
514
		}

515
		FAIL("Unexpected Location_type: " << static_cast<int>(type));
516
517
518
519
	}

	auto Asset_manager::load_raw(const AID& id) -> util::maybe<istream> {
		Location_type type;
520
		std::string   path;
521
522
		std::tie(type, path) = _locate(id);

523
		if(type != Location_type::file)
524
525
526
527
528
529
530
531
532
533
			return util::nothing;

		return _open(path, id);
	}
	auto Asset_manager::save_raw(const AID& id) -> ostream {
		_assets.erase(id);
		return _create(id);
	}

	auto Asset_manager::find_by_path(const std::string& path) -> util::maybe<AID> {
534
535
		static const auto working_dir  = util::replace(pwd(), "\\", "/");
		auto              path_cleared = util::replace(util::replace(path, "\\", "/"), working_dir + "/", "");
536
537
538
539

		for(auto& aid_path : _dispatcher) {
			auto loc = physical_location(aid_path.first, false);
			if(loc.is_some()) {
540
				if(loc.get_or_throw() == path_cleared) {
541
542
543
544
545
					return util::justCopy(aid_path.first);
				}
			}
		}

546
		DEBUG("Couldn't finde asset for '" << path_cleared << "'");
547
548
549
550
551
552
553
554
555
556
557

		return util::nothing;
	}

	auto Asset_manager::watch(AID aid, std::function<void(const AID&)> on_mod) -> uint32_t {
		auto id = _next_watch_id++;
		_watchlist.emplace_back(id, aid, std::move(on_mod));
		return id;
	}

	void Asset_manager::unwatch(uint32_t id) {
558
		auto iter = std::find_if(_watchlist.begin(), _watchlist.end(), [id](auto& w) { return w.id == id; });
559

560
		if(iter != _watchlist.end()) {
561
562
563
564
565
566
567
568
569
570
			if(&_watchlist.back() != &*iter) {
				_watchlist.back() = std::move(*iter);
			}

			_watchlist.pop_back();
		}
	}

	void Asset_manager::_check_watch_entry(Watch_entry& w) {
		Location_type type;
571
		std::string   location;
572
573
		std::tie(type, location) = _locate(w.aid);

574
		if(type == Location_type::file) {
575
			auto last_mod = PHYSFS_getLastModTime(location.c_str());
576
			if(last_mod != -1 && last_mod > w.last_modified) {
577
578
579
580
581
582
				w.last_modified = last_mod;
				w.on_mod(w.aid);
			}
		}
	}
}