Commit 1b35c571 authored by Florian Oetke's avatar Florian Oetke
Browse files

working mulit-threaded asset manager

parent 7182f5d5
......@@ -92,6 +92,11 @@ public:
{
return internal_task != nullptr;
}
std::size_t refcount() const
{
return internal_task!=nullptr ? internal_task->ref_count.load() : 0;
}
// Query whether the task has finished executing
bool ready() const
......
......@@ -26,17 +26,17 @@ namespace mirrage {
, _renderer_factory(std::make_unique<renderer::Deferred_renderer_factory>(
graphics_context(),
window(),
util::make_vector(
renderer::make_pass_factory<renderer::Shadowmapping_pass_factory>(),
renderer::make_pass_factory<renderer::Deferred_pass_factory>(),
renderer::make_pass_factory<renderer::Gen_mipmap_pass_factory>(),
renderer::make_pass_factory<renderer::Ssao_pass_factory>(),
renderer::make_pass_factory<renderer::Gi_pass_factory>(),
renderer::make_pass_factory<renderer::Taa_pass_factory>(),
renderer::make_pass_factory<renderer::Tone_mapping_pass_factory>(),
renderer::make_pass_factory<renderer::Bloom_pass_factory>(),
renderer::make_pass_factory<renderer::Blit_pass_factory>(),
renderer::make_pass_factory<renderer::Gui_pass_factory>()))) {}
assets(),
util::make_vector(renderer::make_pass_factory<renderer::Shadowmapping_pass_factory>(),
renderer::make_pass_factory<renderer::Deferred_pass_factory>(),
renderer::make_pass_factory<renderer::Gen_mipmap_pass_factory>(),
renderer::make_pass_factory<renderer::Ssao_pass_factory>(),
renderer::make_pass_factory<renderer::Gi_pass_factory>(),
renderer::make_pass_factory<renderer::Taa_pass_factory>(),
renderer::make_pass_factory<renderer::Tone_mapping_pass_factory>(),
renderer::make_pass_factory<renderer::Bloom_pass_factory>(),
renderer::make_pass_factory<renderer::Blit_pass_factory>(),
renderer::make_pass_factory<renderer::Gui_pass_factory>()))) {}
Game_engine::~Game_engine() {
screens().clear(); // destroy all screens before the engine
......
......@@ -55,7 +55,7 @@ int main(int argc, char** argv, char** env) {
}
namespace {
constexpr auto app_name = "BachelorProject";
constexpr auto app_name = "Mirrage";
int argc;
char** argv;
char** env;
......
......@@ -12,7 +12,7 @@ namespace mirrage {
Meta_system::Meta_system(Game_engine& engine)
: _entities(engine.assets(), this)
, _renderer(engine.renderer_factory().create_renderer(_entities, *this))
, _model_loading(std::make_unique<renderer::Loading_system>(_entities, _renderer->model_loader()))
, _model_loading(std::make_unique<renderer::Loading_system>(_entities, engine.assets()))
, _nims(std::make_unique<systems::Nim_system>(_entities)) {
_entities.register_component_type<ecs::components::Transform_comp>();
}
......
......@@ -85,7 +85,7 @@ namespace mirrage::systems {
extern void load(sf2::JsonDeserializer& s, Nim_sequence& e);
extern void save(sf2::JsonSerializer& s, const Nim_sequence& e);
using Nim_sequence_ptr = std::shared_ptr<const Nim_sequence>;
using Nim_sequence_ptr = asset::Ptr<Nim_sequence>;
// Manages recording and playback of non-interactive-movies
......
......@@ -134,7 +134,7 @@ namespace mirrage {
.load_maybe<systems::Nim_sequence>("nim:demo_animation"_aid, false)
.process([&](auto&& rec) {
_selected_preset = 0;
_meta_system.nims().play_looped(rec.get());
_meta_system.nims().play_looped(rec);
});
break;
case "pause"_strid:
......@@ -304,6 +304,8 @@ namespace mirrage {
void Test_screen::_draw() {
if(_show_ui) {
_gui.start_frame();
_draw_settings_window();
if(_show_profiler) {
......
......@@ -28,7 +28,8 @@ namespace mirrage::asset {
bool operator==(const AID& o) const noexcept;
bool operator!=(const AID& o) const noexcept;
bool operator<(const AID& o) const noexcept;
operator bool() const noexcept;
operator bool() const noexcept;
auto str() const noexcept -> std::string;
auto type() const noexcept { return _type; }
......
......@@ -52,37 +52,33 @@ namespace mirrage::asset {
class Ptr {
public:
Ptr() = default;
Ptr(const AID& id, std::shared_ptr<R> res = std::shared_ptr<R>());
Ptr(const AID& aid, async::shared_task<R> task) : _aid(aid), _task(std::move(task)) {}
bool operator==(const Ptr& o) const noexcept;
bool operator<(const Ptr& o) const noexcept;
bool operator==(const Ptr& o) const noexcept { return _aid == o._aid; }
bool operator!=(const Ptr& o) const noexcept { return _aid != o._aid; }
bool operator<(const Ptr& o) const noexcept { return _aid < o._aid; }
auto operator*() -> const R&;
auto operator*() const -> const R&;
auto operator-> () -> const R*;
auto operator-> () const -> const R*;
operator bool() const noexcept { return !!_ptr; }
operator std::shared_ptr<const R>() const;
auto operator*() const -> const R& { return get_blocking(); }
auto operator-> () const -> const R* { return &get_blocking(); }
explicit operator bool() const noexcept { return !!aid(); }
auto aid() const noexcept -> const AID& { return _aid; }
void reset() { _ptr.reset(); }
auto get_blocking() const -> const R&;
auto get_if_ready() const -> util::maybe<R&>;
auto ready() const -> bool;
auto internal_task() const -> auto& { return _task; }
void reset();
private:
friend class detail::Asset_container<R>;
AID _aid;
std::shared_ptr<R> _ptr;
AID _aid;
async::shared_task<R> _task;
mutable const R* _cached_result = nullptr;
};
template <typename T>
using Loading = async::shared_task<Ptr<T>>;
template <typename T>
auto make_ready_asset(const AID& id, T&& val) -> Loading<T> {
return async::make_task(Ptr<T>(id, std::make_shared<T>(std::move(val)))).share();
auto make_ready_asset(const AID& id, T&& val) -> Ptr<std::remove_cv_t<std::remove_reference_t<T>>> {
return {id, async::make_task(std::forward<T>(val)).share()};
}
namespace detail {
......@@ -109,7 +105,7 @@ namespace mirrage::asset {
using Loader<T>::load;
using Loader<T>::save;
auto load(AID aid, const std::string& name, bool cache) -> Loading<T>;
auto load(AID aid, const std::string& name, bool cache) -> Ptr<T>;
void save(const AID& aid, const std::string& name, const T&);
......@@ -118,8 +114,9 @@ namespace mirrage::asset {
private:
struct Asset {
Loading<T> ptr;
int64_t last_modified;
AID aid;
async::shared_task<T> task;
int64_t last_modified;
};
Asset_manager& _manager;
......@@ -141,10 +138,10 @@ namespace mirrage::asset {
template <typename T>
auto load(const AID& id, bool cache = true) -> Loading<T>;
auto load(const AID& id, bool cache = true) -> Ptr<T>;
template <typename T>
auto load_maybe(const AID& id, bool cache = true) -> util::maybe<Loading<T>>;
auto load_maybe(const AID& id, bool cache = true) -> util::maybe<Ptr<T>>;
template <typename T>
void save(const AID& id, const T& asset);
......@@ -162,7 +159,7 @@ namespace mirrage::asset {
auto list(Asset_type type) -> std::vector<AID>;
auto last_modified(const AID& id) const noexcept -> util::maybe<std::int64_t>;
auto resolve(const AID& id) const noexcept -> util::maybe<std::string>;
auto resolve(const AID& id, bool only_preexisting = true) const noexcept -> util::maybe<std::string>;
auto resolve_reverse(std::string_view) -> util::maybe<AID>;
template <typename T, typename... Args>
......
......@@ -9,40 +9,31 @@
namespace mirrage::asset {
template <class R>
Ptr<R>::Ptr(const AID& id, std::shared_ptr<R> res) : _aid(id), _ptr(res) {}
auto Ptr<R>::get_blocking() const -> const R& {
if(_cached_result)
return *_cached_result;
template <class R>
const R& Ptr<R>::operator*() {
return *_ptr.get();
}
template <class R>
const R& Ptr<R>::operator*() const {
MIRRAGE_INVARIANT(*this, "Access to unloaded resource");
return *_ptr.get();
return *(_cached_result = &_task.get());
}
template <class R>
const R* Ptr<R>::operator->() {
return _ptr.get();
}
template <class R>
const R* Ptr<R>::operator->() const {
MIRRAGE_INVARIANT(*this, "Access to unloaded resource");
return _ptr.get();
}
auto Ptr<R>::get_if_ready() const -> util::maybe<R&> {
if(_cached_result)
return util::justPtr(_cached_result);
template <class R>
bool Ptr<R>::operator==(const Ptr& o) const noexcept {
return _aid == o._aid;
return ready() ? util::nothing : get_blocking();
}
template <class R>
bool Ptr<R>::operator<(const Ptr& o) const noexcept {
return _aid < o._aid;
auto Ptr<R>::ready() const -> bool {
return _task.ready();
}
template <class R>
Ptr<R>::operator std::shared_ptr<const R>() const {
return _ptr;
void Ptr<R>::reset() {
_aid = {};
_task = {};
_cached_result = nullptr;
}
......@@ -73,46 +64,34 @@ namespace mirrage::asset {
::async::task<TaskType>> || std::is_same_v<T, ::async::shared_task<TaskType>>;
template <typename T>
auto Asset_container<T>::load(AID aid, const std::string& path, bool cache) -> Loading<T> {
auto Asset_container<T>::load(AID aid, const std::string& path, bool cache) -> Ptr<T> {
auto lock = std::scoped_lock{_container_mutex};
auto found = _assets.find(path);
if(found != _assets.end())
return found->second.ptr;
return {aid, found->second.task};
// not found => load
// clang-format off
auto loading = async::spawn([path = std::string(path), aid, this] {
auto result = this->load(_manager._open(aid, path));
using RT = std::remove_reference_t<std::decay_t<decltype(result)>>;
if constexpr(is_task_v<std::shared_ptr<T>, RT>) {
// if the loader returned a task, warp it and return the task (unwrapping)
return result.then([aid](auto&& r) {
return Ptr<T>(aid, std::forward<decltype(r)>(r));
});
} else {
static_assert(std::is_same_v<std::shared_ptr<T>, RT>);
return Ptr<T>(aid, std::move(result));
};
return Loader<T>::load(_manager._open(aid, path));
}).share();
// clang-format on
if(cache)
_assets.try_emplace(path, Asset{loading, _manager._last_modified(path)});
_assets.try_emplace(path, Asset{aid, loading, _manager._last_modified(path)});
return loading;
return {aid, loading};
}
template <typename T>
void Asset_container<T>::save(const AID& aid, const std::string& name, const T& obj) {
auto lock = std::scoped_lock{_container_mutex};
this->save(_manager._open_rw(aid, name), obj);
Loader<T>::save(_manager._open_rw(aid, name), obj);
auto found = _assets.find(name);
if(found != _assets.end() && &*found.value().ptr.get() != &obj) {
if(found != _assets.end() && &found.value().task.get() != &obj) {
_reload_asset(found.value(), found.key()); // replace existing value
}
}
......@@ -121,40 +100,44 @@ namespace mirrage::asset {
void Asset_container<T>::shrink_to_fit() noexcept {
auto lock = std::scoped_lock{_container_mutex};
util::erase_if(_assets, [](const auto& v) { return v.second.ptr.get()._ptr.use_count() <= 1; });
util::erase_if(_assets, [](const auto& v) { return v.second.task.refcount() <= 1; });
}
template <typename T>
void Asset_container<T>::reload() {
auto lock = std::scoped_lock{_container_mutex};
for(auto iter = _assets.begin(); iter != _assets.end(); iter++) {
auto&& key = iter.key();
auto&& value = iter.value();
for(auto && [key, value] : _assets) {
auto last_mod = _manager._last_modified(key);
if(last_mod > value.last_modified) {
_reload_asset(value, key);
_reload_asset(const_cast<Asset&>(value), key);
}
}
}
// TODO: test if this actually works
template <typename T>
void Asset_container<T>::_reload_asset(Asset& asset, const std::string& path) {
auto& asset_ptr = asset.ptr.get();
auto& old_value = const_cast<T&>(asset.task.get());
asset.last_modified = _manager._last_modified(path);
if constexpr(has_reload_v<T>) {
this->reload(_manager._open(asset_ptr.aid(), path), *asset_ptr);
Loader<T>::reload(_manager._open(asset.aid, path), old_value);
} else {
static_assert(std::is_same_v<T&, decltype(*asset_ptr._ptr)>,
"The lhs should be an l-value reference!");
auto new_value = Loader<T>::load(_manager._open(asset.aid, path));
if constexpr(std::is_same_v<decltype(new_value), async::task<T>>) {
old_value = std::move(new_value.get());
*asset_ptr._ptr =
std::move(const_cast<T&>(*load(_manager._open(asset_ptr.aid(), path)).get()));
} else if constexpr(std::is_same_v<decltype(new_value), async::shared_task<T>>) {
old_value = std::move(const_cast<T&>(new_value.get()));
} else {
old_value = std::move(new_value);
}
}
// TODO: notify other systems about change
......@@ -163,16 +146,16 @@ namespace mirrage::asset {
template <typename T>
auto Asset_manager::load(const AID& id, bool cache) -> Loading<T> {
auto Asset_manager::load(const AID& id, bool cache) -> Ptr<T> {
auto a = load_maybe<T>(id, cache);
if(a.is_nothing())
throw std::system_error(Asset_error::resolve_failed);
throw std::system_error(Asset_error::resolve_failed, id.str());
return a.get_or_throw();
}
template <typename T>
auto Asset_manager::load_maybe(const AID& id, bool cache) -> util::maybe<Loading<T>> {
auto Asset_manager::load_maybe(const AID& id, bool cache) -> util::maybe<Ptr<T>> {
auto path = resolve(id);
if(path.is_nothing())
return util::nothing;
......@@ -184,13 +167,13 @@ namespace mirrage::asset {
template <typename T>
void Asset_manager::save(const AID& id, const T& asset) {
auto path = resolve(id);
auto path = resolve(id, false);
if(path.is_nothing())
throw std::system_error(Asset_error::resolve_failed);
throw std::system_error(Asset_error::resolve_failed, id.str());
auto container = _find_container<T>();
if(container.is_nothing())
throw std::system_error(Asset_error::stateful_loader_not_initialized);
throw std::system_error(Asset_error::stateful_loader_not_initialized, id.str());
container.get_or_throw().save(id, path.get_or_throw(), asset);
}
......
......@@ -92,8 +92,8 @@ namespace mirrage::asset {
static_assert(sf2::is_loadable<T, sf2::format::Json_reader>::value,
"Required AssetLoader specialization not provided.");
static auto load(istream in) -> std::shared_ptr<T> {
auto r = std::make_shared<T>();
static auto load(istream in) -> T {
auto r = T();
sf2::deserialize_json(in,
[&](auto& msg, uint32_t row, uint32_t column) {
......@@ -101,7 +101,7 @@ namespace mirrage::asset {
<< row << ":" << column << ": "
<< msg);
},
*r);
r);
return r;
}
......@@ -121,7 +121,7 @@ namespace mirrage::asset {
struct Loader {
static_assert(util::dependent_false<T>(), "Required AssetLoader specialization not provided.");
static auto load(istream in) -> std::shared_ptr<T>;
static auto load(istream in) -> T;
static void save(ostream out, const T& asset);
};
} // namespace mirrage::asset
......@@ -133,7 +133,7 @@ namespace mirrage::asset {
template <>
struct Loader<Bytes> {
static auto load(istream in) -> std::shared_ptr<Bytes> { return std::make_shared<Bytes>(in.bytes()); }
static auto load(istream in) -> Bytes { return in.bytes(); }
void save(ostream out, const Bytes& data) {
out.write(data.data(), gsl::narrow<std::streamsize>(data.size()));
}
......
......@@ -80,8 +80,7 @@ namespace {
auto wildcard = last_of(file, '*').get_or(file.length());
if(wildcard != (file.find_first_of('*') + 1)) {
MIRRAGE_WARN("More than one wildcard ist currently not supported. Found in: "
<< wildcard_path);
MIRRAGE_WARN("More than one wildcard ist currently not supported. Found in: " << wildcard_path);
}
auto prefix = file.substr(0, wildcard - 1);
......@@ -96,21 +95,21 @@ namespace {
}
bool exists_file(const std::string path) {
if(PHYSFS_exists(path.c_str()) == 0)
if(!PHYSFS_exists(path.c_str()))
return false;
auto stat = PHYSFS_Stat{};
if(PHYSFS_stat(path.c_str(), &stat) == 0)
if(!PHYSFS_stat(path.c_str(), &stat))
return false;
return stat.filetype == PHYSFS_FILETYPE_REGULAR;
}
bool exists_dir(const std::string path) {
if(PHYSFS_exists(path.c_str()) == 0)
if(!PHYSFS_exists(path.c_str()))
return false;
auto stat = PHYSFS_Stat{};
if(PHYSFS_stat(path.c_str(), &stat) == 0)
if(!PHYSFS_stat(path.c_str(), &stat))
return false;
return stat.filetype == PHYSFS_FILETYPE_DIRECTORY;
......@@ -132,8 +131,7 @@ namespace {
}
}
constexpr auto default_source = {std::make_tuple("assets", false),
std::make_tuple("assets.zip", true)};
constexpr auto default_source = {std::make_tuple("assets", false), std::make_tuple("assets.zip", true)};
} // namespace
namespace mirrage::asset {
......@@ -227,7 +225,7 @@ namespace mirrage::asset {
"Error adding custom archive: "s + path);
};
if(exists_file("archives.lst")) {
if(!exists_file("archives.lst")) {
bool lost = true;
for(auto& s : default_source) {
const char* path;
......@@ -261,6 +259,7 @@ namespace mirrage::asset {
for(auto&& l : in.lines()) {
if(l.find_last_of('*') != std::string::npos) {
for(auto& file : list_wildcard_files(l)) {
MIRRAGE_INFO("Added FS directory: " << file);
add_source(file.c_str());
}
continue;
......@@ -275,8 +274,8 @@ namespace mirrage::asset {
Asset_manager::~Asset_manager() {
_containers.clear();
if(!PHYSFS_deinit()) {
MIRRAGE_FAIL("Unable to shutdown PhysicsFS: "
<< PHYSFS_getErrorByCode((PHYSFS_getLastErrorCode())));
MIRRAGE_FAIL(
"Unable to shutdown PhysicsFS: " << PHYSFS_getErrorByCode((PHYSFS_getLastErrorCode())));
}
}
......@@ -312,8 +311,7 @@ namespace mirrage::asset {
return resolve(id).process(false, [](auto&& path) { return exists_file(path); });
}
auto Asset_manager::try_delete(const AID& id) -> bool {
return resolve(id).process(true,
[](auto&& path) { return PHYSFS_delete(path.c_str()) == 0; });
return resolve(id).process(true, [](auto&& path) { return PHYSFS_delete(path.c_str()) == 0; });
}
auto Asset_manager::open(const AID& id) -> util::maybe<istream> {
......@@ -364,12 +362,13 @@ namespace mirrage::asset {
return resolve(id).process([&](auto& path) { return _last_modified(path); });
}
auto Asset_manager::resolve(const AID& id) const noexcept -> util::maybe<std::string> {
auto Asset_manager::resolve(const AID& id, bool only_preexisting) const noexcept
-> util::maybe<std::string> {
auto lock = std::shared_lock{_dispatchers_mutex};
auto res = _dispatchers.find(id);
if(res != _dispatchers.end() && exists_file(res->second))
if(res != _dispatchers.end() && (exists_file(res->second) || !only_preexisting))
return res->second;
else if(exists_file(id.name()))
......@@ -382,6 +381,15 @@ namespace mirrage::asset {
auto path = append_file(baseDir.get_or_throw(), id.name());
if(exists_file(path))
return std::move(path);
else if(!only_preexisting) {
PHYSFS_mkdir(baseDir.get_or_throw().c_str());
return std::move(path);
}
}
if(!only_preexisting) {
return id.name();
}
return util::nothing;
......@@ -425,6 +433,7 @@ namespace mirrage::asset {
_dispatchers.clear();
for(auto&& df : list_files("", "assets", ".map")) {
MIRRAGE_INFO("Added asset mapping: " << df);
auto in = _open({}, df);
for(auto&& l : in.lines()) {
auto kvp = util::split(l, "=");
......@@ -438,8 +447,8 @@ namespace mirrage::asset {
auto Asset_manager::_last_modified(const std::string& path) const -> int64_t {
auto stat = PHYSFS_Stat{};
if(auto errc = PHYSFS_stat(path.c_str(), &stat); errc != 0)
throw std::system_error(static_cast<Asset_error>(errc));
if(!PHYSFS_stat(path.c_str(), &stat))
throw std::system_error(static_cast<Asset_error>(PHYSFS_getLastErrorCode()));
return stat.modtime;
}
......