mirror of
https://github.com/shadps4-emu/shadPS4.git
synced 2025-06-02 08:43:16 +00:00
Switch remaining CRLF terminated files to LF
This commit is contained in:
Daniel R.
parent
2c0f986c52
commit
c284cf72e1
28 changed files with 4856 additions and 4856 deletions
|
|
@ -1,203 +1,203 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "common/types.h"
|
||||
#include "video_core/amdgpu/resource.h"
|
||||
#include "video_core/renderer_vulkan/vk_common.h"
|
||||
|
||||
namespace Vulkan {
|
||||
class Instance;
|
||||
class Scheduler;
|
||||
} // namespace Vulkan
|
||||
|
||||
VK_DEFINE_HANDLE(VmaAllocation)
|
||||
VK_DEFINE_HANDLE(VmaAllocator)
|
||||
|
||||
struct VmaAllocationInfo;
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
/// Hints and requirements for the backing memory type of a commit
|
||||
enum class MemoryUsage {
|
||||
DeviceLocal, ///< Requests device local buffer.
|
||||
Upload, ///< Requires a host visible memory type optimized for CPU to GPU uploads
|
||||
Download, ///< Requires a host visible memory type optimized for GPU to CPU readbacks
|
||||
Stream, ///< Requests device local host visible buffer, falling back host memory.
|
||||
};
|
||||
|
||||
constexpr vk::BufferUsageFlags ReadFlags =
|
||||
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eUniformTexelBuffer |
|
||||
vk::BufferUsageFlagBits::eUniformBuffer | vk::BufferUsageFlagBits::eIndexBuffer |
|
||||
vk::BufferUsageFlagBits::eVertexBuffer | vk::BufferUsageFlagBits::eIndirectBuffer;
|
||||
|
||||
constexpr vk::BufferUsageFlags AllFlags = ReadFlags | vk::BufferUsageFlagBits::eTransferDst |
|
||||
vk::BufferUsageFlagBits::eStorageTexelBuffer |
|
||||
vk::BufferUsageFlagBits::eStorageBuffer;
|
||||
|
||||
struct UniqueBuffer {
|
||||
explicit UniqueBuffer(vk::Device device, VmaAllocator allocator);
|
||||
~UniqueBuffer();
|
||||
|
||||
UniqueBuffer(const UniqueBuffer&) = delete;
|
||||
UniqueBuffer& operator=(const UniqueBuffer&) = delete;
|
||||
|
||||
UniqueBuffer(UniqueBuffer&& other)
|
||||
: allocator{std::exchange(other.allocator, VK_NULL_HANDLE)},
|
||||
allocation{std::exchange(other.allocation, VK_NULL_HANDLE)},
|
||||
buffer{std::exchange(other.buffer, VK_NULL_HANDLE)} {}
|
||||
UniqueBuffer& operator=(UniqueBuffer&& other) {
|
||||
buffer = std::exchange(other.buffer, VK_NULL_HANDLE);
|
||||
allocator = std::exchange(other.allocator, VK_NULL_HANDLE);
|
||||
allocation = std::exchange(other.allocation, VK_NULL_HANDLE);
|
||||
return *this;
|
||||
}
|
||||
|
||||
void Create(const vk::BufferCreateInfo& image_ci, MemoryUsage usage,
|
||||
VmaAllocationInfo* out_alloc_info);
|
||||
|
||||
operator vk::Buffer() const {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
vk::Device device;
|
||||
VmaAllocator allocator;
|
||||
VmaAllocation allocation;
|
||||
vk::Buffer buffer{};
|
||||
};
|
||||
|
||||
class Buffer {
|
||||
public:
|
||||
explicit Buffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||
MemoryUsage usage, VAddr cpu_addr_, vk::BufferUsageFlags flags,
|
||||
u64 size_bytes_);
|
||||
|
||||
Buffer& operator=(const Buffer&) = delete;
|
||||
Buffer(const Buffer&) = delete;
|
||||
|
||||
Buffer& operator=(Buffer&&) = default;
|
||||
Buffer(Buffer&&) = default;
|
||||
|
||||
vk::BufferView View(u32 offset, u32 size, bool is_written, AmdGpu::DataFormat dfmt,
|
||||
AmdGpu::NumberFormat nfmt);
|
||||
|
||||
/// Increases the likeliness of this being a stream buffer
|
||||
void IncreaseStreamScore(int score) noexcept {
|
||||
stream_score += score;
|
||||
}
|
||||
|
||||
/// Returns the likeliness of this being a stream buffer
|
||||
[[nodiscard]] int StreamScore() const noexcept {
|
||||
return stream_score;
|
||||
}
|
||||
|
||||
/// Returns true when vaddr -> vaddr+size is fully contained in the buffer
|
||||
[[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept {
|
||||
return addr >= cpu_addr && addr + size <= cpu_addr + SizeBytes();
|
||||
}
|
||||
|
||||
/// Returns the base CPU address of the buffer
|
||||
[[nodiscard]] VAddr CpuAddr() const noexcept {
|
||||
return cpu_addr;
|
||||
}
|
||||
|
||||
/// Returns the offset relative to the given CPU address
|
||||
[[nodiscard]] u32 Offset(VAddr other_cpu_addr) const noexcept {
|
||||
return static_cast<u32>(other_cpu_addr - cpu_addr);
|
||||
}
|
||||
|
||||
size_t SizeBytes() const {
|
||||
return size_bytes;
|
||||
}
|
||||
|
||||
vk::Buffer Handle() const noexcept {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
std::optional<vk::BufferMemoryBarrier2> GetBarrier(vk::AccessFlagBits2 dst_acess_mask,
|
||||
vk::PipelineStageFlagBits2 dst_stage) {
|
||||
if (dst_acess_mask == access_mask && stage == dst_stage) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto barrier = vk::BufferMemoryBarrier2{
|
||||
.srcStageMask = stage,
|
||||
.srcAccessMask = access_mask,
|
||||
.dstStageMask = dst_stage,
|
||||
.dstAccessMask = dst_acess_mask,
|
||||
.buffer = buffer.buffer,
|
||||
.size = size_bytes,
|
||||
};
|
||||
access_mask = dst_acess_mask;
|
||||
stage = dst_stage;
|
||||
return barrier;
|
||||
}
|
||||
|
||||
public:
|
||||
VAddr cpu_addr = 0;
|
||||
bool is_picked{};
|
||||
bool is_coherent{};
|
||||
bool is_deleted{};
|
||||
int stream_score = 0;
|
||||
size_t size_bytes = 0;
|
||||
std::span<u8> mapped_data;
|
||||
const Vulkan::Instance* instance;
|
||||
Vulkan::Scheduler* scheduler;
|
||||
MemoryUsage usage;
|
||||
UniqueBuffer buffer;
|
||||
vk::AccessFlagBits2 access_mask{vk::AccessFlagBits2::eNone};
|
||||
vk::PipelineStageFlagBits2 stage{vk::PipelineStageFlagBits2::eNone};
|
||||
};
|
||||
|
||||
class StreamBuffer : public Buffer {
|
||||
public:
|
||||
explicit StreamBuffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||
MemoryUsage usage, u64 size_bytes_);
|
||||
|
||||
/// Reserves a region of memory from the stream buffer.
|
||||
std::pair<u8*, u64> Map(u64 size, u64 alignment = 0);
|
||||
|
||||
/// Ensures that reserved bytes of memory are available to the GPU.
|
||||
void Commit();
|
||||
|
||||
/// Maps and commits a memory region with user provided data
|
||||
u64 Copy(VAddr src, size_t size, size_t alignment = 0) {
|
||||
const auto [data, offset] = Map(size, alignment);
|
||||
std::memcpy(data, reinterpret_cast<const void*>(src), size);
|
||||
Commit();
|
||||
return offset;
|
||||
}
|
||||
|
||||
u64 GetFreeSize() const {
|
||||
return size_bytes - offset - mapped_size;
|
||||
}
|
||||
|
||||
private:
|
||||
struct Watch {
|
||||
u64 tick{};
|
||||
u64 upper_bound{};
|
||||
};
|
||||
|
||||
/// Increases the amount of watches available.
|
||||
void ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size);
|
||||
|
||||
/// Waits pending watches until requested upper bound.
|
||||
void WaitPendingOperations(u64 requested_upper_bound);
|
||||
|
||||
private:
|
||||
u64 offset{};
|
||||
u64 mapped_size{};
|
||||
std::vector<Watch> current_watches;
|
||||
std::size_t current_watch_cursor{};
|
||||
std::optional<size_t> invalidation_mark;
|
||||
std::vector<Watch> previous_watches;
|
||||
std::size_t wait_cursor{};
|
||||
u64 wait_bound{};
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "common/types.h"
|
||||
#include "video_core/amdgpu/resource.h"
|
||||
#include "video_core/renderer_vulkan/vk_common.h"
|
||||
|
||||
namespace Vulkan {
|
||||
class Instance;
|
||||
class Scheduler;
|
||||
} // namespace Vulkan
|
||||
|
||||
VK_DEFINE_HANDLE(VmaAllocation)
|
||||
VK_DEFINE_HANDLE(VmaAllocator)
|
||||
|
||||
struct VmaAllocationInfo;
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
/// Hints and requirements for the backing memory type of a commit
|
||||
enum class MemoryUsage {
|
||||
DeviceLocal, ///< Requests device local buffer.
|
||||
Upload, ///< Requires a host visible memory type optimized for CPU to GPU uploads
|
||||
Download, ///< Requires a host visible memory type optimized for GPU to CPU readbacks
|
||||
Stream, ///< Requests device local host visible buffer, falling back host memory.
|
||||
};
|
||||
|
||||
constexpr vk::BufferUsageFlags ReadFlags =
|
||||
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eUniformTexelBuffer |
|
||||
vk::BufferUsageFlagBits::eUniformBuffer | vk::BufferUsageFlagBits::eIndexBuffer |
|
||||
vk::BufferUsageFlagBits::eVertexBuffer | vk::BufferUsageFlagBits::eIndirectBuffer;
|
||||
|
||||
constexpr vk::BufferUsageFlags AllFlags = ReadFlags | vk::BufferUsageFlagBits::eTransferDst |
|
||||
vk::BufferUsageFlagBits::eStorageTexelBuffer |
|
||||
vk::BufferUsageFlagBits::eStorageBuffer;
|
||||
|
||||
struct UniqueBuffer {
|
||||
explicit UniqueBuffer(vk::Device device, VmaAllocator allocator);
|
||||
~UniqueBuffer();
|
||||
|
||||
UniqueBuffer(const UniqueBuffer&) = delete;
|
||||
UniqueBuffer& operator=(const UniqueBuffer&) = delete;
|
||||
|
||||
UniqueBuffer(UniqueBuffer&& other)
|
||||
: allocator{std::exchange(other.allocator, VK_NULL_HANDLE)},
|
||||
allocation{std::exchange(other.allocation, VK_NULL_HANDLE)},
|
||||
buffer{std::exchange(other.buffer, VK_NULL_HANDLE)} {}
|
||||
UniqueBuffer& operator=(UniqueBuffer&& other) {
|
||||
buffer = std::exchange(other.buffer, VK_NULL_HANDLE);
|
||||
allocator = std::exchange(other.allocator, VK_NULL_HANDLE);
|
||||
allocation = std::exchange(other.allocation, VK_NULL_HANDLE);
|
||||
return *this;
|
||||
}
|
||||
|
||||
void Create(const vk::BufferCreateInfo& image_ci, MemoryUsage usage,
|
||||
VmaAllocationInfo* out_alloc_info);
|
||||
|
||||
operator vk::Buffer() const {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
vk::Device device;
|
||||
VmaAllocator allocator;
|
||||
VmaAllocation allocation;
|
||||
vk::Buffer buffer{};
|
||||
};
|
||||
|
||||
class Buffer {
|
||||
public:
|
||||
explicit Buffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||
MemoryUsage usage, VAddr cpu_addr_, vk::BufferUsageFlags flags,
|
||||
u64 size_bytes_);
|
||||
|
||||
Buffer& operator=(const Buffer&) = delete;
|
||||
Buffer(const Buffer&) = delete;
|
||||
|
||||
Buffer& operator=(Buffer&&) = default;
|
||||
Buffer(Buffer&&) = default;
|
||||
|
||||
vk::BufferView View(u32 offset, u32 size, bool is_written, AmdGpu::DataFormat dfmt,
|
||||
AmdGpu::NumberFormat nfmt);
|
||||
|
||||
/// Increases the likeliness of this being a stream buffer
|
||||
void IncreaseStreamScore(int score) noexcept {
|
||||
stream_score += score;
|
||||
}
|
||||
|
||||
/// Returns the likeliness of this being a stream buffer
|
||||
[[nodiscard]] int StreamScore() const noexcept {
|
||||
return stream_score;
|
||||
}
|
||||
|
||||
/// Returns true when vaddr -> vaddr+size is fully contained in the buffer
|
||||
[[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept {
|
||||
return addr >= cpu_addr && addr + size <= cpu_addr + SizeBytes();
|
||||
}
|
||||
|
||||
/// Returns the base CPU address of the buffer
|
||||
[[nodiscard]] VAddr CpuAddr() const noexcept {
|
||||
return cpu_addr;
|
||||
}
|
||||
|
||||
/// Returns the offset relative to the given CPU address
|
||||
[[nodiscard]] u32 Offset(VAddr other_cpu_addr) const noexcept {
|
||||
return static_cast<u32>(other_cpu_addr - cpu_addr);
|
||||
}
|
||||
|
||||
size_t SizeBytes() const {
|
||||
return size_bytes;
|
||||
}
|
||||
|
||||
vk::Buffer Handle() const noexcept {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
std::optional<vk::BufferMemoryBarrier2> GetBarrier(vk::AccessFlagBits2 dst_acess_mask,
|
||||
vk::PipelineStageFlagBits2 dst_stage) {
|
||||
if (dst_acess_mask == access_mask && stage == dst_stage) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto barrier = vk::BufferMemoryBarrier2{
|
||||
.srcStageMask = stage,
|
||||
.srcAccessMask = access_mask,
|
||||
.dstStageMask = dst_stage,
|
||||
.dstAccessMask = dst_acess_mask,
|
||||
.buffer = buffer.buffer,
|
||||
.size = size_bytes,
|
||||
};
|
||||
access_mask = dst_acess_mask;
|
||||
stage = dst_stage;
|
||||
return barrier;
|
||||
}
|
||||
|
||||
public:
|
||||
VAddr cpu_addr = 0;
|
||||
bool is_picked{};
|
||||
bool is_coherent{};
|
||||
bool is_deleted{};
|
||||
int stream_score = 0;
|
||||
size_t size_bytes = 0;
|
||||
std::span<u8> mapped_data;
|
||||
const Vulkan::Instance* instance;
|
||||
Vulkan::Scheduler* scheduler;
|
||||
MemoryUsage usage;
|
||||
UniqueBuffer buffer;
|
||||
vk::AccessFlagBits2 access_mask{vk::AccessFlagBits2::eNone};
|
||||
vk::PipelineStageFlagBits2 stage{vk::PipelineStageFlagBits2::eNone};
|
||||
};
|
||||
|
||||
class StreamBuffer : public Buffer {
|
||||
public:
|
||||
explicit StreamBuffer(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||
MemoryUsage usage, u64 size_bytes_);
|
||||
|
||||
/// Reserves a region of memory from the stream buffer.
|
||||
std::pair<u8*, u64> Map(u64 size, u64 alignment = 0);
|
||||
|
||||
/// Ensures that reserved bytes of memory are available to the GPU.
|
||||
void Commit();
|
||||
|
||||
/// Maps and commits a memory region with user provided data
|
||||
u64 Copy(VAddr src, size_t size, size_t alignment = 0) {
|
||||
const auto [data, offset] = Map(size, alignment);
|
||||
std::memcpy(data, reinterpret_cast<const void*>(src), size);
|
||||
Commit();
|
||||
return offset;
|
||||
}
|
||||
|
||||
u64 GetFreeSize() const {
|
||||
return size_bytes - offset - mapped_size;
|
||||
}
|
||||
|
||||
private:
|
||||
struct Watch {
|
||||
u64 tick{};
|
||||
u64 upper_bound{};
|
||||
};
|
||||
|
||||
/// Increases the amount of watches available.
|
||||
void ReserveWatches(std::vector<Watch>& watches, std::size_t grow_size);
|
||||
|
||||
/// Waits pending watches until requested upper bound.
|
||||
void WaitPendingOperations(u64 requested_upper_bound);
|
||||
|
||||
private:
|
||||
u64 offset{};
|
||||
u64 mapped_size{};
|
||||
std::vector<Watch> current_watches;
|
||||
std::size_t current_watch_cursor{};
|
||||
std::optional<size_t> invalidation_mark;
|
||||
std::vector<Watch> previous_watches;
|
||||
std::size_t wait_cursor{};
|
||||
u64 wait_bound{};
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,168 +1,168 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <boost/container/small_vector.hpp>
|
||||
#include <boost/icl/interval_map.hpp>
|
||||
#include <tsl/robin_map.h>
|
||||
#include "common/div_ceil.h"
|
||||
#include "common/slot_vector.h"
|
||||
#include "common/types.h"
|
||||
#include "video_core/buffer_cache/buffer.h"
|
||||
#include "video_core/buffer_cache/memory_tracker_base.h"
|
||||
#include "video_core/buffer_cache/range_set.h"
|
||||
#include "video_core/multi_level_page_table.h"
|
||||
|
||||
namespace AmdGpu {
|
||||
struct Liverpool;
|
||||
}
|
||||
|
||||
namespace Shader {
|
||||
namespace Gcn {
|
||||
struct FetchShaderData;
|
||||
}
|
||||
struct Info;
|
||||
} // namespace Shader
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
using BufferId = Common::SlotId;
|
||||
|
||||
static constexpr BufferId NULL_BUFFER_ID{0};
|
||||
|
||||
class TextureCache;
|
||||
|
||||
class BufferCache {
|
||||
public:
|
||||
static constexpr u32 CACHING_PAGEBITS = 12;
|
||||
static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS;
|
||||
static constexpr u64 DEVICE_PAGESIZE = 4_KB;
|
||||
|
||||
struct Traits {
|
||||
using Entry = BufferId;
|
||||
static constexpr size_t AddressSpaceBits = 40;
|
||||
static constexpr size_t FirstLevelBits = 14;
|
||||
static constexpr size_t PageBits = CACHING_PAGEBITS;
|
||||
};
|
||||
using PageTable = MultiLevelPageTable<Traits>;
|
||||
|
||||
struct OverlapResult {
|
||||
boost::container::small_vector<BufferId, 16> ids;
|
||||
VAddr begin;
|
||||
VAddr end;
|
||||
bool has_stream_leap = false;
|
||||
};
|
||||
|
||||
public:
|
||||
explicit BufferCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||
AmdGpu::Liverpool* liverpool, TextureCache& texture_cache,
|
||||
PageManager& tracker);
|
||||
~BufferCache();
|
||||
|
||||
/// Returns a pointer to GDS device local buffer.
|
||||
[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept {
|
||||
return &gds_buffer;
|
||||
}
|
||||
|
||||
/// Retrieves the buffer with the specified id.
|
||||
[[nodiscard]] Buffer& GetBuffer(BufferId id) {
|
||||
return slot_buffers[id];
|
||||
}
|
||||
|
||||
[[nodiscard]] vk::BufferView& NullBufferView() {
|
||||
return null_buffer_view;
|
||||
}
|
||||
|
||||
/// Invalidates any buffer in the logical page range.
|
||||
void InvalidateMemory(VAddr device_addr, u64 size);
|
||||
|
||||
/// Binds host vertex buffers for the current draw.
|
||||
bool BindVertexBuffers(const Shader::Info& vs_info,
|
||||
const std::optional<Shader::Gcn::FetchShaderData>& fetch_shader);
|
||||
|
||||
/// Bind host index buffer for the current draw.
|
||||
u32 BindIndexBuffer(bool& is_indexed, u32 index_offset);
|
||||
|
||||
/// Writes a value to GPU buffer.
|
||||
void InlineData(VAddr address, const void* value, u32 num_bytes, bool is_gds);
|
||||
|
||||
[[nodiscard]] std::pair<Buffer*, u32> ObtainHostUBO(std::span<const u32> data);
|
||||
|
||||
/// Obtains a buffer for the specified region.
|
||||
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(VAddr gpu_addr, u32 size, bool is_written,
|
||||
bool is_texel_buffer = false,
|
||||
BufferId buffer_id = {});
|
||||
|
||||
/// Attempts to obtain a buffer without modifying the cache contents.
|
||||
[[nodiscard]] std::pair<Buffer*, u32> ObtainViewBuffer(VAddr gpu_addr, u32 size,
|
||||
bool prefer_gpu);
|
||||
|
||||
/// Return true when a region is registered on the cache
|
||||
[[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
|
||||
|
||||
/// Return true when a CPU region is modified from the CPU
|
||||
[[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
|
||||
|
||||
/// Return true when a CPU region is modified from the GPU
|
||||
[[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
|
||||
|
||||
[[nodiscard]] BufferId FindBuffer(VAddr device_addr, u32 size);
|
||||
|
||||
private:
|
||||
template <typename Func>
|
||||
void ForEachBufferInRange(VAddr device_addr, u64 size, Func&& func) {
|
||||
const u64 page_end = Common::DivCeil(device_addr + size, CACHING_PAGESIZE);
|
||||
for (u64 page = device_addr >> CACHING_PAGEBITS; page < page_end;) {
|
||||
const BufferId buffer_id = page_table[page];
|
||||
if (!buffer_id) {
|
||||
++page;
|
||||
continue;
|
||||
}
|
||||
Buffer& buffer = slot_buffers[buffer_id];
|
||||
func(buffer_id, buffer);
|
||||
|
||||
const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
|
||||
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
|
||||
}
|
||||
}
|
||||
|
||||
void DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 size);
|
||||
|
||||
[[nodiscard]] OverlapResult ResolveOverlaps(VAddr device_addr, u32 wanted_size);
|
||||
|
||||
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
|
||||
|
||||
[[nodiscard]] BufferId CreateBuffer(VAddr device_addr, u32 wanted_size);
|
||||
|
||||
void Register(BufferId buffer_id);
|
||||
|
||||
void Unregister(BufferId buffer_id);
|
||||
|
||||
template <bool insert>
|
||||
void ChangeRegister(BufferId buffer_id);
|
||||
|
||||
void SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size, bool is_texel_buffer);
|
||||
|
||||
bool SynchronizeBufferFromImage(Buffer& buffer, VAddr device_addr, u32 size);
|
||||
|
||||
void DeleteBuffer(BufferId buffer_id);
|
||||
|
||||
const Vulkan::Instance& instance;
|
||||
Vulkan::Scheduler& scheduler;
|
||||
AmdGpu::Liverpool* liverpool;
|
||||
TextureCache& texture_cache;
|
||||
PageManager& tracker;
|
||||
StreamBuffer staging_buffer;
|
||||
StreamBuffer stream_buffer;
|
||||
Buffer gds_buffer;
|
||||
std::mutex mutex;
|
||||
Common::SlotVector<Buffer> slot_buffers;
|
||||
RangeSet gpu_modified_ranges;
|
||||
vk::BufferView null_buffer_view;
|
||||
MemoryTracker memory_tracker;
|
||||
PageTable page_table;
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <boost/container/small_vector.hpp>
|
||||
#include <boost/icl/interval_map.hpp>
|
||||
#include <tsl/robin_map.h>
|
||||
#include "common/div_ceil.h"
|
||||
#include "common/slot_vector.h"
|
||||
#include "common/types.h"
|
||||
#include "video_core/buffer_cache/buffer.h"
|
||||
#include "video_core/buffer_cache/memory_tracker_base.h"
|
||||
#include "video_core/buffer_cache/range_set.h"
|
||||
#include "video_core/multi_level_page_table.h"
|
||||
|
||||
namespace AmdGpu {
|
||||
struct Liverpool;
|
||||
}
|
||||
|
||||
namespace Shader {
|
||||
namespace Gcn {
|
||||
struct FetchShaderData;
|
||||
}
|
||||
struct Info;
|
||||
} // namespace Shader
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
using BufferId = Common::SlotId;
|
||||
|
||||
static constexpr BufferId NULL_BUFFER_ID{0};
|
||||
|
||||
class TextureCache;
|
||||
|
||||
class BufferCache {
|
||||
public:
|
||||
static constexpr u32 CACHING_PAGEBITS = 12;
|
||||
static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS;
|
||||
static constexpr u64 DEVICE_PAGESIZE = 4_KB;
|
||||
|
||||
struct Traits {
|
||||
using Entry = BufferId;
|
||||
static constexpr size_t AddressSpaceBits = 40;
|
||||
static constexpr size_t FirstLevelBits = 14;
|
||||
static constexpr size_t PageBits = CACHING_PAGEBITS;
|
||||
};
|
||||
using PageTable = MultiLevelPageTable<Traits>;
|
||||
|
||||
struct OverlapResult {
|
||||
boost::container::small_vector<BufferId, 16> ids;
|
||||
VAddr begin;
|
||||
VAddr end;
|
||||
bool has_stream_leap = false;
|
||||
};
|
||||
|
||||
public:
|
||||
explicit BufferCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
|
||||
AmdGpu::Liverpool* liverpool, TextureCache& texture_cache,
|
||||
PageManager& tracker);
|
||||
~BufferCache();
|
||||
|
||||
/// Returns a pointer to GDS device local buffer.
|
||||
[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept {
|
||||
return &gds_buffer;
|
||||
}
|
||||
|
||||
/// Retrieves the buffer with the specified id.
|
||||
[[nodiscard]] Buffer& GetBuffer(BufferId id) {
|
||||
return slot_buffers[id];
|
||||
}
|
||||
|
||||
[[nodiscard]] vk::BufferView& NullBufferView() {
|
||||
return null_buffer_view;
|
||||
}
|
||||
|
||||
/// Invalidates any buffer in the logical page range.
|
||||
void InvalidateMemory(VAddr device_addr, u64 size);
|
||||
|
||||
/// Binds host vertex buffers for the current draw.
|
||||
bool BindVertexBuffers(const Shader::Info& vs_info,
|
||||
const std::optional<Shader::Gcn::FetchShaderData>& fetch_shader);
|
||||
|
||||
/// Bind host index buffer for the current draw.
|
||||
u32 BindIndexBuffer(bool& is_indexed, u32 index_offset);
|
||||
|
||||
/// Writes a value to GPU buffer.
|
||||
void InlineData(VAddr address, const void* value, u32 num_bytes, bool is_gds);
|
||||
|
||||
[[nodiscard]] std::pair<Buffer*, u32> ObtainHostUBO(std::span<const u32> data);
|
||||
|
||||
/// Obtains a buffer for the specified region.
|
||||
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(VAddr gpu_addr, u32 size, bool is_written,
|
||||
bool is_texel_buffer = false,
|
||||
BufferId buffer_id = {});
|
||||
|
||||
/// Attempts to obtain a buffer without modifying the cache contents.
|
||||
[[nodiscard]] std::pair<Buffer*, u32> ObtainViewBuffer(VAddr gpu_addr, u32 size,
|
||||
bool prefer_gpu);
|
||||
|
||||
/// Return true when a region is registered on the cache
|
||||
[[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
|
||||
|
||||
/// Return true when a CPU region is modified from the CPU
|
||||
[[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
|
||||
|
||||
/// Return true when a CPU region is modified from the GPU
|
||||
[[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
|
||||
|
||||
[[nodiscard]] BufferId FindBuffer(VAddr device_addr, u32 size);
|
||||
|
||||
private:
|
||||
template <typename Func>
|
||||
void ForEachBufferInRange(VAddr device_addr, u64 size, Func&& func) {
|
||||
const u64 page_end = Common::DivCeil(device_addr + size, CACHING_PAGESIZE);
|
||||
for (u64 page = device_addr >> CACHING_PAGEBITS; page < page_end;) {
|
||||
const BufferId buffer_id = page_table[page];
|
||||
if (!buffer_id) {
|
||||
++page;
|
||||
continue;
|
||||
}
|
||||
Buffer& buffer = slot_buffers[buffer_id];
|
||||
func(buffer_id, buffer);
|
||||
|
||||
const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
|
||||
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
|
||||
}
|
||||
}
|
||||
|
||||
void DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 size);
|
||||
|
||||
[[nodiscard]] OverlapResult ResolveOverlaps(VAddr device_addr, u32 wanted_size);
|
||||
|
||||
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
|
||||
|
||||
[[nodiscard]] BufferId CreateBuffer(VAddr device_addr, u32 wanted_size);
|
||||
|
||||
void Register(BufferId buffer_id);
|
||||
|
||||
void Unregister(BufferId buffer_id);
|
||||
|
||||
template <bool insert>
|
||||
void ChangeRegister(BufferId buffer_id);
|
||||
|
||||
void SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size, bool is_texel_buffer);
|
||||
|
||||
bool SynchronizeBufferFromImage(Buffer& buffer, VAddr device_addr, u32 size);
|
||||
|
||||
void DeleteBuffer(BufferId buffer_id);
|
||||
|
||||
const Vulkan::Instance& instance;
|
||||
Vulkan::Scheduler& scheduler;
|
||||
AmdGpu::Liverpool* liverpool;
|
||||
TextureCache& texture_cache;
|
||||
PageManager& tracker;
|
||||
StreamBuffer staging_buffer;
|
||||
StreamBuffer stream_buffer;
|
||||
Buffer gds_buffer;
|
||||
std::mutex mutex;
|
||||
Common::SlotVector<Buffer> slot_buffers;
|
||||
RangeSet gpu_modified_ranges;
|
||||
vk::BufferView null_buffer_view;
|
||||
MemoryTracker memory_tracker;
|
||||
PageTable page_table;
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
|
|
|
|||
|
|
@ -1,175 +1,175 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <deque>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
#include "common/types.h"
|
||||
#include "video_core/buffer_cache/word_manager.h"
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
class MemoryTracker {
|
||||
public:
|
||||
static constexpr size_t MAX_CPU_PAGE_BITS = 40;
|
||||
static constexpr size_t HIGHER_PAGE_BITS = 22;
|
||||
static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
|
||||
static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
|
||||
static constexpr size_t NUM_HIGH_PAGES = 1ULL << (MAX_CPU_PAGE_BITS - HIGHER_PAGE_BITS);
|
||||
static constexpr size_t MANAGER_POOL_SIZE = 32;
|
||||
static constexpr size_t WORDS_STACK_NEEDED = HIGHER_PAGE_SIZE / BYTES_PER_WORD;
|
||||
using Manager = WordManager<WORDS_STACK_NEEDED>;
|
||||
|
||||
public:
|
||||
explicit MemoryTracker(PageManager* tracker_) : tracker{tracker_} {}
|
||||
~MemoryTracker() = default;
|
||||
|
||||
/// Returns true if a region has been modified from the CPU
|
||||
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||
return IteratePages<true>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template IsRegionModified<Type::CPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns true if a region has been modified from the GPU
|
||||
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||
return IteratePages<false>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template IsRegionModified<Type::GPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Mark region as CPU modified, notifying the device_tracker about this change
|
||||
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::CPU, true>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Unmark region as CPU modified, notifying the device_tracker about this change
|
||||
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::CPU, false>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Mark region as modified from the host GPU
|
||||
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::GPU, true>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Unmark region as modified from the host GPU
|
||||
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::GPU, false>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
|
||||
template <typename Func>
|
||||
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||
IteratePages<true>(query_cpu_range, query_size,
|
||||
[&func](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ForEachModifiedRange<Type::CPU, true>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
});
|
||||
}
|
||||
|
||||
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
|
||||
template <bool clear, typename Func>
|
||||
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||
IteratePages<false>(query_cpu_range, query_size,
|
||||
[&func](Manager* manager, u64 offset, size_t size) {
|
||||
if constexpr (clear) {
|
||||
manager->template ForEachModifiedRange<Type::GPU, true>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
} else {
|
||||
manager->template ForEachModifiedRange<Type::GPU, false>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
/**
|
||||
* @brief IteratePages Iterates L2 word manager page table.
|
||||
* @param cpu_address Start byte cpu address
|
||||
* @param size Size in bytes of the region of iterate.
|
||||
* @param func Callback for each word manager.
|
||||
* @return
|
||||
*/
|
||||
template <bool create_region_on_fail, typename Func>
|
||||
bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
|
||||
using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
|
||||
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||
std::size_t remaining_size{size};
|
||||
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
|
||||
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
|
||||
while (remaining_size > 0) {
|
||||
const std::size_t copy_amount{
|
||||
std::min<std::size_t>(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
|
||||
auto* manager{top_tier[page_index]};
|
||||
if (manager) {
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(manager, page_offset, copy_amount)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
func(manager, page_offset, copy_amount);
|
||||
}
|
||||
} else if constexpr (create_region_on_fail) {
|
||||
CreateRegion(page_index);
|
||||
manager = top_tier[page_index];
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(manager, page_offset, copy_amount)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
func(manager, page_offset, copy_amount);
|
||||
}
|
||||
}
|
||||
page_index++;
|
||||
page_offset = 0;
|
||||
remaining_size -= copy_amount;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void CreateRegion(std::size_t page_index) {
|
||||
const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
|
||||
if (free_managers.empty()) {
|
||||
manager_pool.emplace_back();
|
||||
auto& last_pool = manager_pool.back();
|
||||
for (size_t i = 0; i < MANAGER_POOL_SIZE; i++) {
|
||||
std::construct_at(&last_pool[i], tracker, 0, HIGHER_PAGE_SIZE);
|
||||
free_managers.push_back(&last_pool[i]);
|
||||
}
|
||||
}
|
||||
// Each manager tracks a 4_MB virtual address space.
|
||||
auto* new_manager = free_managers.back();
|
||||
new_manager->SetCpuAddress(base_cpu_addr);
|
||||
free_managers.pop_back();
|
||||
top_tier[page_index] = new_manager;
|
||||
}
|
||||
|
||||
PageManager* tracker;
|
||||
std::deque<std::array<Manager, MANAGER_POOL_SIZE>> manager_pool;
|
||||
std::vector<Manager*> free_managers;
|
||||
std::array<Manager*, NUM_HIGH_PAGES> top_tier{};
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <deque>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
#include "common/types.h"
|
||||
#include "video_core/buffer_cache/word_manager.h"
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
class MemoryTracker {
|
||||
public:
|
||||
static constexpr size_t MAX_CPU_PAGE_BITS = 40;
|
||||
static constexpr size_t HIGHER_PAGE_BITS = 22;
|
||||
static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
|
||||
static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
|
||||
static constexpr size_t NUM_HIGH_PAGES = 1ULL << (MAX_CPU_PAGE_BITS - HIGHER_PAGE_BITS);
|
||||
static constexpr size_t MANAGER_POOL_SIZE = 32;
|
||||
static constexpr size_t WORDS_STACK_NEEDED = HIGHER_PAGE_SIZE / BYTES_PER_WORD;
|
||||
using Manager = WordManager<WORDS_STACK_NEEDED>;
|
||||
|
||||
public:
|
||||
explicit MemoryTracker(PageManager* tracker_) : tracker{tracker_} {}
|
||||
~MemoryTracker() = default;
|
||||
|
||||
/// Returns true if a region has been modified from the CPU
|
||||
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||
return IteratePages<true>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template IsRegionModified<Type::CPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns true if a region has been modified from the GPU
|
||||
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||
return IteratePages<false>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template IsRegionModified<Type::GPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Mark region as CPU modified, notifying the device_tracker about this change
|
||||
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::CPU, true>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Unmark region as CPU modified, notifying the device_tracker about this change
|
||||
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::CPU, false>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Mark region as modified from the host GPU
|
||||
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::GPU, true>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Unmark region as modified from the host GPU
|
||||
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::GPU, false>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
|
||||
template <typename Func>
|
||||
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||
IteratePages<true>(query_cpu_range, query_size,
|
||||
[&func](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ForEachModifiedRange<Type::CPU, true>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
});
|
||||
}
|
||||
|
||||
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
|
||||
template <bool clear, typename Func>
|
||||
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||
IteratePages<false>(query_cpu_range, query_size,
|
||||
[&func](Manager* manager, u64 offset, size_t size) {
|
||||
if constexpr (clear) {
|
||||
manager->template ForEachModifiedRange<Type::GPU, true>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
} else {
|
||||
manager->template ForEachModifiedRange<Type::GPU, false>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
/**
|
||||
* @brief IteratePages Iterates L2 word manager page table.
|
||||
* @param cpu_address Start byte cpu address
|
||||
* @param size Size in bytes of the region of iterate.
|
||||
* @param func Callback for each word manager.
|
||||
* @return
|
||||
*/
|
||||
template <bool create_region_on_fail, typename Func>
|
||||
bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
|
||||
using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
|
||||
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||
std::size_t remaining_size{size};
|
||||
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
|
||||
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
|
||||
while (remaining_size > 0) {
|
||||
const std::size_t copy_amount{
|
||||
std::min<std::size_t>(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
|
||||
auto* manager{top_tier[page_index]};
|
||||
if (manager) {
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(manager, page_offset, copy_amount)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
func(manager, page_offset, copy_amount);
|
||||
}
|
||||
} else if constexpr (create_region_on_fail) {
|
||||
CreateRegion(page_index);
|
||||
manager = top_tier[page_index];
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(manager, page_offset, copy_amount)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
func(manager, page_offset, copy_amount);
|
||||
}
|
||||
}
|
||||
page_index++;
|
||||
page_offset = 0;
|
||||
remaining_size -= copy_amount;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void CreateRegion(std::size_t page_index) {
|
||||
const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
|
||||
if (free_managers.empty()) {
|
||||
manager_pool.emplace_back();
|
||||
auto& last_pool = manager_pool.back();
|
||||
for (size_t i = 0; i < MANAGER_POOL_SIZE; i++) {
|
||||
std::construct_at(&last_pool[i], tracker, 0, HIGHER_PAGE_SIZE);
|
||||
free_managers.push_back(&last_pool[i]);
|
||||
}
|
||||
}
|
||||
// Each manager tracks a 4_MB virtual address space.
|
||||
auto* new_manager = free_managers.back();
|
||||
new_manager->SetCpuAddress(base_cpu_addr);
|
||||
free_managers.pop_back();
|
||||
top_tier[page_index] = new_manager;
|
||||
}
|
||||
|
||||
PageManager* tracker;
|
||||
std::deque<std::array<Manager, MANAGER_POOL_SIZE>> manager_pool;
|
||||
std::vector<Manager*> free_managers;
|
||||
std::array<Manager*, NUM_HIGH_PAGES> top_tier{};
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
|
|
|
|||
|
|
@ -1,398 +1,398 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <span>
|
||||
#include <utility>
|
||||
#include "common/div_ceil.h"
|
||||
#include "common/types.h"
|
||||
#include "video_core/page_manager.h"
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
constexpr u64 PAGES_PER_WORD = 64;
|
||||
constexpr u64 BYTES_PER_PAGE = 4_KB;
|
||||
constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
|
||||
|
||||
enum class Type {
|
||||
CPU,
|
||||
GPU,
|
||||
Untracked,
|
||||
};
|
||||
|
||||
/// Vector tracking modified pages tightly packed with small vector optimization
|
||||
template <size_t stack_words = 1>
|
||||
struct WordsArray {
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
|
||||
return is_short ? stack.data() : heap;
|
||||
}
|
||||
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
|
||||
return is_short ? stack.data() : heap;
|
||||
}
|
||||
|
||||
std::array<u64, stack_words> stack{}; ///< Small buffers storage
|
||||
u64* heap; ///< Not-small buffers pointer to the storage
|
||||
};
|
||||
|
||||
template <size_t stack_words = 1>
|
||||
struct Words {
|
||||
explicit Words() = default;
|
||||
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
|
||||
num_words = Common::DivCeil(size_bytes, BYTES_PER_WORD);
|
||||
if (IsShort()) {
|
||||
cpu.stack.fill(~u64{0});
|
||||
gpu.stack.fill(0);
|
||||
untracked.stack.fill(~u64{0});
|
||||
} else {
|
||||
// Share allocation between CPU and GPU pages and set their default values
|
||||
u64* const alloc = new u64[num_words * 3];
|
||||
cpu.heap = alloc;
|
||||
gpu.heap = alloc + num_words;
|
||||
untracked.heap = alloc + num_words * 2;
|
||||
std::fill_n(cpu.heap, num_words, ~u64{0});
|
||||
std::fill_n(gpu.heap, num_words, 0);
|
||||
std::fill_n(untracked.heap, num_words, ~u64{0});
|
||||
}
|
||||
// Clean up tailing bits
|
||||
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
|
||||
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
|
||||
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
|
||||
const u64 last_word = (~u64{0} << shift) >> shift;
|
||||
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
}
|
||||
|
||||
~Words() {
|
||||
Release();
|
||||
}
|
||||
|
||||
Words& operator=(Words&& rhs) noexcept {
|
||||
Release();
|
||||
size_bytes = rhs.size_bytes;
|
||||
num_words = rhs.num_words;
|
||||
cpu = rhs.cpu;
|
||||
gpu = rhs.gpu;
|
||||
untracked = rhs.untracked;
|
||||
rhs.cpu.heap = nullptr;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Words(Words&& rhs) noexcept
|
||||
: size_bytes{rhs.size_bytes}, num_words{rhs.num_words}, cpu{rhs.cpu}, gpu{rhs.gpu},
|
||||
untracked{rhs.untracked} {
|
||||
rhs.cpu.heap = nullptr;
|
||||
}
|
||||
|
||||
Words& operator=(const Words&) = delete;
|
||||
Words(const Words&) = delete;
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return num_words <= stack_words;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the buffer
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return num_words;
|
||||
}
|
||||
|
||||
/// Release buffer resources
|
||||
void Release() {
|
||||
if (!IsShort()) {
|
||||
// CPU written words is the base for the heap allocation
|
||||
delete[] cpu.heap;
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
std::span<u64> Span() noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return std::span<u64>(cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return std::span<u64>(gpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return std::span<u64>(untracked.Pointer(IsShort()), num_words);
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
std::span<const u64> Span() const noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return std::span<const u64>(cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return std::span<const u64>(gpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return std::span<const u64>(untracked.Pointer(IsShort()), num_words);
|
||||
}
|
||||
}
|
||||
|
||||
u64 size_bytes = 0;
|
||||
size_t num_words = 0;
|
||||
WordsArray<stack_words> cpu;
|
||||
WordsArray<stack_words> gpu;
|
||||
WordsArray<stack_words> untracked;
|
||||
};
|
||||
|
||||
template <size_t stack_words = 1>
|
||||
class WordManager {
|
||||
public:
|
||||
explicit WordManager(PageManager* tracker_, VAddr cpu_addr_, u64 size_bytes)
|
||||
: tracker{tracker_}, cpu_addr{cpu_addr_}, words{size_bytes} {}
|
||||
|
||||
explicit WordManager() = default;
|
||||
|
||||
void SetCpuAddress(VAddr new_cpu_addr) {
|
||||
cpu_addr = new_cpu_addr;
|
||||
}
|
||||
|
||||
VAddr GetCpuAddr() const {
|
||||
return cpu_addr;
|
||||
}
|
||||
|
||||
static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
|
||||
constexpr size_t number_bits = sizeof(u64) * 8;
|
||||
const size_t limit_page_end = number_bits - std::min(page_end, number_bits);
|
||||
u64 bits = (word >> page_start) << page_start;
|
||||
bits = (bits << limit_page_end) >> limit_page_end;
|
||||
return bits;
|
||||
}
|
||||
|
||||
static std::pair<size_t, size_t> GetWordPage(VAddr address) {
|
||||
const size_t converted_address = static_cast<size_t>(address);
|
||||
const size_t word_number = converted_address / BYTES_PER_WORD;
|
||||
const size_t amount_pages = converted_address % BYTES_PER_WORD;
|
||||
return std::make_pair(word_number, amount_pages / BYTES_PER_PAGE);
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void IterateWords(size_t offset, size_t size, Func&& func) const {
|
||||
using FuncReturn = std::invoke_result_t<Func, std::size_t, u64>;
|
||||
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||
const size_t start = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset), 0LL));
|
||||
const size_t end = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset + size), 0LL));
|
||||
if (start >= SizeBytes() || end <= start) {
|
||||
return;
|
||||
}
|
||||
auto [start_word, start_page] = GetWordPage(start);
|
||||
auto [end_word, end_page] = GetWordPage(end + BYTES_PER_PAGE - 1ULL);
|
||||
const size_t num_words = NumWords();
|
||||
start_word = std::min(start_word, num_words);
|
||||
end_word = std::min(end_word, num_words);
|
||||
const size_t diff = end_word - start_word;
|
||||
end_word += (end_page + PAGES_PER_WORD - 1ULL) / PAGES_PER_WORD;
|
||||
end_word = std::min(end_word, num_words);
|
||||
end_page += diff * PAGES_PER_WORD;
|
||||
constexpr u64 base_mask{~0ULL};
|
||||
for (size_t word_index = start_word; word_index < end_word; word_index++) {
|
||||
const u64 mask = ExtractBits(base_mask, start_page, end_page);
|
||||
start_page = 0;
|
||||
end_page -= PAGES_PER_WORD;
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(word_index, mask)) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
func(word_index, mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void IteratePages(u64 mask, Func&& func) const {
|
||||
size_t offset = 0;
|
||||
while (mask != 0) {
|
||||
const size_t empty_bits = std::countr_zero(mask);
|
||||
offset += empty_bits;
|
||||
mask = mask >> empty_bits;
|
||||
|
||||
const size_t continuous_bits = std::countr_one(mask);
|
||||
func(offset, continuous_bits);
|
||||
mask = continuous_bits < PAGES_PER_WORD ? (mask >> continuous_bits) : 0;
|
||||
offset += continuous_bits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Change the state of a range of pages
|
||||
*
|
||||
* @param dirty_addr Base address to mark or unmark as modified
|
||||
* @param size Size in bytes to mark or unmark as modified
|
||||
*/
|
||||
template <Type type, bool enable>
|
||||
void ChangeRegionState(u64 dirty_addr, u64 size) noexcept(type == Type::GPU) {
|
||||
std::span<u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||
IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) {
|
||||
if constexpr (type == Type::CPU) {
|
||||
NotifyPageTracker<!enable>(index, untracked_words[index], mask);
|
||||
}
|
||||
if constexpr (enable) {
|
||||
state_words[index] |= mask;
|
||||
if constexpr (type == Type::CPU) {
|
||||
untracked_words[index] |= mask;
|
||||
}
|
||||
} else {
|
||||
state_words[index] &= ~mask;
|
||||
if constexpr (type == Type::CPU) {
|
||||
untracked_words[index] &= ~mask;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Loop over each page in the given range, turn off those bits and notify the tracker if
|
||||
* needed. Call the given function on each turned off range.
|
||||
*
|
||||
* @param query_cpu_range Base CPU address to loop over
|
||||
* @param size Size in bytes of the CPU range to loop over
|
||||
* @param func Function to call for each turned off region
|
||||
*/
|
||||
template <Type type, bool clear, typename Func>
|
||||
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
std::span<u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||
const size_t offset = query_cpu_range - cpu_addr;
|
||||
bool pending = false;
|
||||
size_t pending_offset{};
|
||||
size_t pending_pointer{};
|
||||
const auto release = [&]() {
|
||||
func(cpu_addr + pending_offset * BYTES_PER_PAGE,
|
||||
(pending_pointer - pending_offset) * BYTES_PER_PAGE);
|
||||
};
|
||||
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||
if constexpr (type == Type::GPU) {
|
||||
mask &= ~untracked_words[index];
|
||||
}
|
||||
const u64 word = state_words[index] & mask;
|
||||
if constexpr (clear) {
|
||||
if constexpr (type == Type::CPU) {
|
||||
NotifyPageTracker<true>(index, untracked_words[index], mask);
|
||||
}
|
||||
state_words[index] &= ~mask;
|
||||
if constexpr (type == Type::CPU) {
|
||||
untracked_words[index] &= ~mask;
|
||||
}
|
||||
}
|
||||
const size_t base_offset = index * PAGES_PER_WORD;
|
||||
IteratePages(word, [&](size_t pages_offset, size_t pages_size) {
|
||||
const auto reset = [&]() {
|
||||
pending_offset = base_offset + pages_offset;
|
||||
pending_pointer = base_offset + pages_offset + pages_size;
|
||||
};
|
||||
if (!pending) {
|
||||
reset();
|
||||
pending = true;
|
||||
return;
|
||||
}
|
||||
if (pending_pointer == base_offset + pages_offset) {
|
||||
pending_pointer += pages_size;
|
||||
return;
|
||||
}
|
||||
release();
|
||||
reset();
|
||||
});
|
||||
});
|
||||
if (pending) {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a region has been modified
|
||||
*
|
||||
* @param offset Offset in bytes from the start of the buffer
|
||||
* @param size Size in bytes of the region to query for modifications
|
||||
*/
|
||||
template <Type type>
|
||||
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
const std::span<const u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] const std::span<const u64> untracked_words =
|
||||
words.template Span<Type::Untracked>();
|
||||
bool result = false;
|
||||
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||
if constexpr (type == Type::GPU) {
|
||||
mask &= ~untracked_words[index];
|
||||
}
|
||||
const u64 word = state_words[index] & mask;
|
||||
if (word != 0) {
|
||||
result = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the manager
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return words.NumWords();
|
||||
}
|
||||
|
||||
/// Returns the size in bytes of the manager
|
||||
[[nodiscard]] u64 SizeBytes() const noexcept {
|
||||
return words.size_bytes;
|
||||
}
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return words.IsShort();
|
||||
}
|
||||
|
||||
private:
|
||||
template <Type type>
|
||||
u64* Array() noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
const u64* Array() const noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify tracker about changes in the CPU tracking state of a word in the buffer
|
||||
*
|
||||
* @param word_index Index to the word to notify to the tracker
|
||||
* @param current_bits Current state of the word
|
||||
* @param new_bits New state of the word
|
||||
*
|
||||
* @tparam add_to_tracker True when the tracker should start tracking the new pages
|
||||
*/
|
||||
template <bool add_to_tracker>
|
||||
void NotifyPageTracker(u64 word_index, u64 current_bits, u64 new_bits) const {
|
||||
u64 changed_bits = (add_to_tracker ? current_bits : ~current_bits) & new_bits;
|
||||
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
|
||||
IteratePages(changed_bits, [&](size_t offset, size_t size) {
|
||||
tracker->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE, size * BYTES_PER_PAGE,
|
||||
add_to_tracker ? 1 : -1);
|
||||
});
|
||||
}
|
||||
|
||||
PageManager* tracker;
|
||||
VAddr cpu_addr = 0;
|
||||
Words<stack_words> words;
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <span>
|
||||
#include <utility>
|
||||
#include "common/div_ceil.h"
|
||||
#include "common/types.h"
|
||||
#include "video_core/page_manager.h"
|
||||
|
||||
namespace VideoCore {
|
||||
|
||||
constexpr u64 PAGES_PER_WORD = 64;
|
||||
constexpr u64 BYTES_PER_PAGE = 4_KB;
|
||||
constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
|
||||
|
||||
enum class Type {
|
||||
CPU,
|
||||
GPU,
|
||||
Untracked,
|
||||
};
|
||||
|
||||
/// Vector tracking modified pages tightly packed with small vector optimization
|
||||
template <size_t stack_words = 1>
|
||||
struct WordsArray {
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
|
||||
return is_short ? stack.data() : heap;
|
||||
}
|
||||
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
|
||||
return is_short ? stack.data() : heap;
|
||||
}
|
||||
|
||||
std::array<u64, stack_words> stack{}; ///< Small buffers storage
|
||||
u64* heap; ///< Not-small buffers pointer to the storage
|
||||
};
|
||||
|
||||
template <size_t stack_words = 1>
|
||||
struct Words {
|
||||
explicit Words() = default;
|
||||
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
|
||||
num_words = Common::DivCeil(size_bytes, BYTES_PER_WORD);
|
||||
if (IsShort()) {
|
||||
cpu.stack.fill(~u64{0});
|
||||
gpu.stack.fill(0);
|
||||
untracked.stack.fill(~u64{0});
|
||||
} else {
|
||||
// Share allocation between CPU and GPU pages and set their default values
|
||||
u64* const alloc = new u64[num_words * 3];
|
||||
cpu.heap = alloc;
|
||||
gpu.heap = alloc + num_words;
|
||||
untracked.heap = alloc + num_words * 2;
|
||||
std::fill_n(cpu.heap, num_words, ~u64{0});
|
||||
std::fill_n(gpu.heap, num_words, 0);
|
||||
std::fill_n(untracked.heap, num_words, ~u64{0});
|
||||
}
|
||||
// Clean up tailing bits
|
||||
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
|
||||
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
|
||||
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
|
||||
const u64 last_word = (~u64{0} << shift) >> shift;
|
||||
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
}
|
||||
|
||||
~Words() {
|
||||
Release();
|
||||
}
|
||||
|
||||
Words& operator=(Words&& rhs) noexcept {
|
||||
Release();
|
||||
size_bytes = rhs.size_bytes;
|
||||
num_words = rhs.num_words;
|
||||
cpu = rhs.cpu;
|
||||
gpu = rhs.gpu;
|
||||
untracked = rhs.untracked;
|
||||
rhs.cpu.heap = nullptr;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Words(Words&& rhs) noexcept
|
||||
: size_bytes{rhs.size_bytes}, num_words{rhs.num_words}, cpu{rhs.cpu}, gpu{rhs.gpu},
|
||||
untracked{rhs.untracked} {
|
||||
rhs.cpu.heap = nullptr;
|
||||
}
|
||||
|
||||
Words& operator=(const Words&) = delete;
|
||||
Words(const Words&) = delete;
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return num_words <= stack_words;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the buffer
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return num_words;
|
||||
}
|
||||
|
||||
/// Release buffer resources
|
||||
void Release() {
|
||||
if (!IsShort()) {
|
||||
// CPU written words is the base for the heap allocation
|
||||
delete[] cpu.heap;
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
std::span<u64> Span() noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return std::span<u64>(cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return std::span<u64>(gpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return std::span<u64>(untracked.Pointer(IsShort()), num_words);
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
std::span<const u64> Span() const noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return std::span<const u64>(cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return std::span<const u64>(gpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return std::span<const u64>(untracked.Pointer(IsShort()), num_words);
|
||||
}
|
||||
}
|
||||
|
||||
u64 size_bytes = 0;
|
||||
size_t num_words = 0;
|
||||
WordsArray<stack_words> cpu;
|
||||
WordsArray<stack_words> gpu;
|
||||
WordsArray<stack_words> untracked;
|
||||
};
|
||||
|
||||
template <size_t stack_words = 1>
|
||||
class WordManager {
|
||||
public:
|
||||
explicit WordManager(PageManager* tracker_, VAddr cpu_addr_, u64 size_bytes)
|
||||
: tracker{tracker_}, cpu_addr{cpu_addr_}, words{size_bytes} {}
|
||||
|
||||
explicit WordManager() = default;
|
||||
|
||||
void SetCpuAddress(VAddr new_cpu_addr) {
|
||||
cpu_addr = new_cpu_addr;
|
||||
}
|
||||
|
||||
VAddr GetCpuAddr() const {
|
||||
return cpu_addr;
|
||||
}
|
||||
|
||||
static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
|
||||
constexpr size_t number_bits = sizeof(u64) * 8;
|
||||
const size_t limit_page_end = number_bits - std::min(page_end, number_bits);
|
||||
u64 bits = (word >> page_start) << page_start;
|
||||
bits = (bits << limit_page_end) >> limit_page_end;
|
||||
return bits;
|
||||
}
|
||||
|
||||
static std::pair<size_t, size_t> GetWordPage(VAddr address) {
|
||||
const size_t converted_address = static_cast<size_t>(address);
|
||||
const size_t word_number = converted_address / BYTES_PER_WORD;
|
||||
const size_t amount_pages = converted_address % BYTES_PER_WORD;
|
||||
return std::make_pair(word_number, amount_pages / BYTES_PER_PAGE);
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void IterateWords(size_t offset, size_t size, Func&& func) const {
|
||||
using FuncReturn = std::invoke_result_t<Func, std::size_t, u64>;
|
||||
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||
const size_t start = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset), 0LL));
|
||||
const size_t end = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset + size), 0LL));
|
||||
if (start >= SizeBytes() || end <= start) {
|
||||
return;
|
||||
}
|
||||
auto [start_word, start_page] = GetWordPage(start);
|
||||
auto [end_word, end_page] = GetWordPage(end + BYTES_PER_PAGE - 1ULL);
|
||||
const size_t num_words = NumWords();
|
||||
start_word = std::min(start_word, num_words);
|
||||
end_word = std::min(end_word, num_words);
|
||||
const size_t diff = end_word - start_word;
|
||||
end_word += (end_page + PAGES_PER_WORD - 1ULL) / PAGES_PER_WORD;
|
||||
end_word = std::min(end_word, num_words);
|
||||
end_page += diff * PAGES_PER_WORD;
|
||||
constexpr u64 base_mask{~0ULL};
|
||||
for (size_t word_index = start_word; word_index < end_word; word_index++) {
|
||||
const u64 mask = ExtractBits(base_mask, start_page, end_page);
|
||||
start_page = 0;
|
||||
end_page -= PAGES_PER_WORD;
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(word_index, mask)) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
func(word_index, mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void IteratePages(u64 mask, Func&& func) const {
|
||||
size_t offset = 0;
|
||||
while (mask != 0) {
|
||||
const size_t empty_bits = std::countr_zero(mask);
|
||||
offset += empty_bits;
|
||||
mask = mask >> empty_bits;
|
||||
|
||||
const size_t continuous_bits = std::countr_one(mask);
|
||||
func(offset, continuous_bits);
|
||||
mask = continuous_bits < PAGES_PER_WORD ? (mask >> continuous_bits) : 0;
|
||||
offset += continuous_bits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Change the state of a range of pages
|
||||
*
|
||||
* @param dirty_addr Base address to mark or unmark as modified
|
||||
* @param size Size in bytes to mark or unmark as modified
|
||||
*/
|
||||
template <Type type, bool enable>
|
||||
void ChangeRegionState(u64 dirty_addr, u64 size) noexcept(type == Type::GPU) {
|
||||
std::span<u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||
IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) {
|
||||
if constexpr (type == Type::CPU) {
|
||||
NotifyPageTracker<!enable>(index, untracked_words[index], mask);
|
||||
}
|
||||
if constexpr (enable) {
|
||||
state_words[index] |= mask;
|
||||
if constexpr (type == Type::CPU) {
|
||||
untracked_words[index] |= mask;
|
||||
}
|
||||
} else {
|
||||
state_words[index] &= ~mask;
|
||||
if constexpr (type == Type::CPU) {
|
||||
untracked_words[index] &= ~mask;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Loop over each page in the given range, turn off those bits and notify the tracker if
|
||||
* needed. Call the given function on each turned off range.
|
||||
*
|
||||
* @param query_cpu_range Base CPU address to loop over
|
||||
* @param size Size in bytes of the CPU range to loop over
|
||||
* @param func Function to call for each turned off region
|
||||
*/
|
||||
template <Type type, bool clear, typename Func>
|
||||
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
std::span<u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||
const size_t offset = query_cpu_range - cpu_addr;
|
||||
bool pending = false;
|
||||
size_t pending_offset{};
|
||||
size_t pending_pointer{};
|
||||
const auto release = [&]() {
|
||||
func(cpu_addr + pending_offset * BYTES_PER_PAGE,
|
||||
(pending_pointer - pending_offset) * BYTES_PER_PAGE);
|
||||
};
|
||||
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||
if constexpr (type == Type::GPU) {
|
||||
mask &= ~untracked_words[index];
|
||||
}
|
||||
const u64 word = state_words[index] & mask;
|
||||
if constexpr (clear) {
|
||||
if constexpr (type == Type::CPU) {
|
||||
NotifyPageTracker<true>(index, untracked_words[index], mask);
|
||||
}
|
||||
state_words[index] &= ~mask;
|
||||
if constexpr (type == Type::CPU) {
|
||||
untracked_words[index] &= ~mask;
|
||||
}
|
||||
}
|
||||
const size_t base_offset = index * PAGES_PER_WORD;
|
||||
IteratePages(word, [&](size_t pages_offset, size_t pages_size) {
|
||||
const auto reset = [&]() {
|
||||
pending_offset = base_offset + pages_offset;
|
||||
pending_pointer = base_offset + pages_offset + pages_size;
|
||||
};
|
||||
if (!pending) {
|
||||
reset();
|
||||
pending = true;
|
||||
return;
|
||||
}
|
||||
if (pending_pointer == base_offset + pages_offset) {
|
||||
pending_pointer += pages_size;
|
||||
return;
|
||||
}
|
||||
release();
|
||||
reset();
|
||||
});
|
||||
});
|
||||
if (pending) {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a region has been modified
|
||||
*
|
||||
* @param offset Offset in bytes from the start of the buffer
|
||||
* @param size Size in bytes of the region to query for modifications
|
||||
*/
|
||||
template <Type type>
|
||||
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
const std::span<const u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] const std::span<const u64> untracked_words =
|
||||
words.template Span<Type::Untracked>();
|
||||
bool result = false;
|
||||
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||
if constexpr (type == Type::GPU) {
|
||||
mask &= ~untracked_words[index];
|
||||
}
|
||||
const u64 word = state_words[index] & mask;
|
||||
if (word != 0) {
|
||||
result = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the manager
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return words.NumWords();
|
||||
}
|
||||
|
||||
/// Returns the size in bytes of the manager
|
||||
[[nodiscard]] u64 SizeBytes() const noexcept {
|
||||
return words.size_bytes;
|
||||
}
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return words.IsShort();
|
||||
}
|
||||
|
||||
private:
|
||||
template <Type type>
|
||||
u64* Array() noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
const u64* Array() const noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify tracker about changes in the CPU tracking state of a word in the buffer
|
||||
*
|
||||
* @param word_index Index to the word to notify to the tracker
|
||||
* @param current_bits Current state of the word
|
||||
* @param new_bits New state of the word
|
||||
*
|
||||
* @tparam add_to_tracker True when the tracker should start tracking the new pages
|
||||
*/
|
||||
template <bool add_to_tracker>
|
||||
void NotifyPageTracker(u64 word_index, u64 current_bits, u64 new_bits) const {
|
||||
u64 changed_bits = (add_to_tracker ? current_bits : ~current_bits) & new_bits;
|
||||
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
|
||||
IteratePages(changed_bits, [&](size_t offset, size_t size) {
|
||||
tracker->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE, size * BYTES_PER_PAGE,
|
||||
add_to_tracker ? 1 : -1);
|
||||
});
|
||||
}
|
||||
|
||||
PageManager* tracker;
|
||||
VAddr cpu_addr = 0;
|
||||
Words<stack_words> words;
|
||||
};
|
||||
|
||||
} // namespace VideoCore
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue