[PATCH v4 6/6] drm/amdgpu: integrate VRAM migration into SVM range map and fault paths

[Junhua Shen <Junhua.Shen@amd.com>]

Wire the migration layer into the SVM range map and GPU fault
call chains:

- Add amdgpu_pagemap_capable() guard in amdgpu_svm_attr_devmem_possible()
  to disable devmem when pagemap is not supported
- Add device_private_page_owner to drm_gpusvm_ctx in both prefetch
  and fault map contexts
- Call amdgpu_svm_range_migrate_range() before GPU mapping in both
  fault and prefetch paths to perform VRAM migration when preferred
- Support AMDGPU_INTERCONNECT_VRAM in amdgpu_svm_range_update_gpu_range()
  by clearing AMDGPU_PTE_SYSTEM and AMDGPU_PTE_SNOOPED flags for
  VRAM pages

Signed-off-by: Junhua Shen <Junhua.Shen@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c  |  4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c |  9 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 21 +++++++++++++++----
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c
index e50b67540c99..115bda12e625 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_attr.c
@@ -25,6 +25,7 @@
 #include "amdgpu_svm.h"
 #include "amdgpu_svm_attr.h"
 #include "amdgpu.h"
+#include "amdgpu_svm_range_migrate.h"
 
 #include <linux/err.h>
 #include <linux/errno.h>
@@ -55,6 +56,9 @@ struct attr_get_ctx {
 bool amdgpu_svm_attr_devmem_possible(struct amdgpu_svm *svm,
 				     const struct amdgpu_svm_attrs *attrs)
 {
+	if (!amdgpu_pagemap_capable(svm))
+		return false;
+
 	if (svm->adev->apu_prefer_gtt)
 		return false;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c
index 7763eb029eaa..b5d21a66a228 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c
@@ -26,6 +26,8 @@
 #include "amdgpu_svm_attr.h"
 #include "amdgpu_svm_fault.h"
 #include "amdgpu_svm_range.h"
+#include "amdgpu_svm_range_migrate.h"
+#include "amdgpu_migrate.h"
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_gmc.h"
@@ -153,13 +155,14 @@ static int amdgpu_svm_range_map_fault(struct amdgpu_svm *svm,
 	const struct amdgpu_svm_attrs *attrs = &attr_range->attrs;
 	bool devmem_possible = amdgpu_svm_attr_devmem_possible(svm, attrs);
 	bool need_vram_migration = amdgpu_svm_attr_prefer_vram(svm, attrs);
-	devmem_possible = false; /* TODO: add migration */
 	struct drm_gpusvm_ctx map_ctx = {
 		.read_only = !!(attrs->flags & AMDGPU_SVM_ATTR_BIT_GPU_RO),
 		.devmem_possible = devmem_possible,
 		.check_pages_threshold = devmem_possible ? SZ_64K : 0,
 		.devmem_only = need_vram_migration && devmem_possible,
 		.timeslice_ms = need_vram_migration && devmem_possible ? 5 : 0,
+		.device_private_page_owner = devmem_possible ?
+			AMDGPU_PGMAP_OWNER(svm->adev) : NULL,
 	};
 	struct amdgpu_svm_range *range;
 	ktime_t timestamp = ktime_get_boottime();
@@ -228,7 +231,9 @@ static int amdgpu_svm_range_map_fault(struct amdgpu_svm *svm,
 	}
 
 	AMDGPU_SVM_RANGE_DEBUG(range, "PAGE FAULT");
-	/* TODO: add migration*/
+	if (need_vram_migration)
+		amdgpu_svm_range_migrate_range(svm, &range->base,
+					      AMDGPU_SVM_MIGRATE_TO_VRAM);
 
 	AMDGPU_SVM_RANGE_DEBUG(range, "GET PAGES");
 	ret = amdgpu_svm_range_get_pages(svm, &range->base, &map_ctx);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
index fe543a16b399..b77f3a52f3ae 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
@@ -26,6 +26,8 @@
 #include "amdgpu_svm_attr.h"
 #include "amdgpu_svm_range.h"
 #include "amdgpu_svm_fault.h"
+#include "amdgpu_svm_range_migrate.h"
+#include "amdgpu_migrate.h"
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
 
@@ -244,9 +246,11 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
 		unsigned long seg_pages = min_t(unsigned long, 1UL << entry->order,
 						npages - mapped_pages);
 		unsigned long start_page, last_page;
+		uint64_t seg_pte_flags = pte_flags;
 		bool is_last_seg;
 
-		if (entry->proto != DRM_INTERCONNECT_SYSTEM)
+		if (entry->proto != DRM_INTERCONNECT_SYSTEM &&
+		    entry->proto != AMDGPU_INTERCONNECT_VRAM)
 			return -EOPNOTSUPP;
 
 		start_page = range_start_page + mapped_pages;
@@ -254,9 +258,13 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
 		mapped_pages += seg_pages;
 		is_last_seg = mapped_pages == npages;
 
+		/* For VRAM pages, clear the SYSTEM and SNOOPED bits */
+		if (entry->proto == AMDGPU_INTERCONNECT_VRAM)
+			seg_pte_flags &= ~(AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED);
+
 		ret = amdgpu_vm_update_range(svm->adev, svm->vm, false, false,
 					     flush_tlb && is_last_seg, true, NULL,
-					     start_page, last_page, pte_flags,
+					     start_page, last_page, seg_pte_flags,
 					     0, entry->addr, NULL, NULL,
 					     wait_fence && is_last_seg ? fence : NULL);
 		if (ret)
@@ -365,12 +373,13 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm,
 	int ret;
 	bool devmem_possible = amdgpu_svm_attr_devmem_possible(svm, attrs);
 	bool need_vram_migration = amdgpu_svm_attr_prefer_vram(svm, attrs);
-	devmem_possible = false; /* TODO: add migration */
 	struct drm_gpusvm_ctx map_ctx = {
 		.read_only = !!(attrs->flags & AMDGPU_SVM_ATTR_BIT_GPU_RO),
 		.devmem_possible = devmem_possible,
 		.devmem_only = need_vram_migration && devmem_possible,
 		.check_pages_threshold = devmem_possible ? SZ_64K : 0,
+		.device_private_page_owner = devmem_possible ?
+			AMDGPU_PGMAP_OWNER(svm->adev) : NULL,
 	};
 
 	while (addr < end) {
@@ -399,7 +408,11 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm,
 			continue;
 		}
 
-		/* TODO: add migration */
+		if (need_vram_migration) {
+			AMDGPU_SVM_RANGE_DEBUG(range, "PREFETCH - MIGRATION PAGES");
+			amdgpu_svm_range_migrate_range(svm, &range->base,
+						      AMDGPU_SVM_MIGRATE_TO_VRAM);
+		}
 
 		AMDGPU_SVM_RANGE_DEBUG(range, "PREFETCH - GET PAGES");
 
-- 
2.34.1