[PATCH v3 5/5] drm/amdgpu: integrate VRAM migration into SVM range map path

[Junhua Shen <Junhua.Shen@amd.com>]

hook up ZONE_DEVICE registration in device init and reset.

Wire the migration decision layer into the SVM range map call chain:

- Add migrate_mode parameter to amdgpu_svm_range_map_attr_ranges()
  and propagate it through map_interval -> per-range map loop
- Call amdgpu_svm_range_migrate_range() on each drm_gpusvm_range
  before GPU mapping to perform VRAM migration or eviction
- Pass appropriate migrate_mode from SVM attr prefetch triggers
  and the restore worker

Signed-off-by: Junhua Shen <Junhua.Shen@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c    |   4 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c     |   4 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c       |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 136 +++++++++---------
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h |   5 +-
 5 files changed, 87 insertions(+), 66 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index fbe553c38583..3be51a2c0106 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -78,6 +78,7 @@
 #include "amdgpu_reset.h"
 #include "amdgpu_virt.h"
 #include "amdgpu_dev_coredump.h"
+#include "amdgpu_migrate.h"
 
 #include <linux/suspend.h>
 #include <drm/task_barrier.h>
@@ -4076,6 +4077,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 
 	/* Don't init kfd if whole hive need to be reset during init */
 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
+#if IS_ENABLED(CONFIG_DRM_AMDGPU_SVM)
+		amdgpu_svm_migration_init(adev);
+#endif
 		kgd2kfd_init_zone_device(adev);
 		kfd_update_svm_support_properties(adev);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
index 28c4ad62f50e..c94d43f3ab42 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
@@ -25,6 +25,7 @@
 #include "aldebaran.h"
 #include "sienna_cichlid.h"
 #include "smu_v13_0_10.h"
+#include "amdgpu_migrate.h"
 
 static int amdgpu_reset_xgmi_reset_on_init_suspend(struct amdgpu_device *adev)
 {
@@ -87,6 +88,9 @@ static int amdgpu_reset_xgmi_reset_on_init_restore_hwctxt(
 		return r;
 	list_for_each_entry(tmp_adev, reset_device_list, reset_list) {
 		if (!tmp_adev->kfd.init_complete) {
+#if IS_ENABLED(CONFIG_DRM_AMDGPU_SVM)
+			amdgpu_svm_migration_init(tmp_adev);
+#endif
 			kgd2kfd_init_zone_device(tmp_adev);
 			amdgpu_amdkfd_device_init(tmp_adev);
 			amdgpu_amdkfd_drm_client_create(tmp_adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
index 57103a140164..cd8dbe56a9e9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
@@ -32,6 +32,7 @@
 #include "amdgpu_svm.h"
 #include "amdgpu_svm_attr.h"
 #include "amdgpu_svm_range.h"
+#include "amdgpu_svm_range_migrate.h"
 #include "amdgpu_vm.h"
 
 #if IS_ENABLED(CONFIG_DRM_AMDGPU_SVM)
@@ -341,7 +342,8 @@ int amdgpu_svm_handle_fault(struct amdgpu_device *adev, uint32_t pasid,
 	AMDGPU_SVM_TRACE("handle_fault: map_attr page=0x%lx\n", fault_page);
 
 	down_write(&svm->svm_lock);
-	ret = amdgpu_svm_range_map_attr_ranges(svm, fault_page, fault_page);
+	ret = amdgpu_svm_range_map_attr_ranges(svm, fault_page, fault_page,
+					       AMDGPU_SVM_MIGRATE_PREFERRED);
 	up_write(&svm->svm_lock);
 
 	if (ret)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
index 472a641fb836..8880e2ba79cc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
@@ -25,6 +25,7 @@
 #include "amdgpu_svm.h"
 #include "amdgpu_svm_attr.h"
 #include "amdgpu_svm_range.h"
+#include "amdgpu_migrate.h"
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_vm.h"
@@ -114,7 +115,6 @@ range_pages_valid(struct amdgpu_svm *svm,
 	return drm_gpusvm_range_pages_valid(&svm->gpusvm, range);
 }
 
-
 static int
 amdgpu_svm_range_gpu_unmap_in_notifier(struct amdgpu_svm *svm,
 				      struct drm_gpusvm_range *range,
@@ -247,11 +247,11 @@ amdgpu_svm_range_attr_pte_flags(struct amdgpu_svm *svm,
 	return pte_flags;
 }
 
-	/*
-	* POC/WA: reuse kfd apis for queue quiesce/resume
-	* But kfd apis are for process level, not for GPU VM level
-	* need consider potential issues
-	*/
+/*
+ * POC/WA: reuse kfd apis for queue quiesce/resume
+ * But kfd apis are for process level, not for GPU VM level
+ * need consider potential issues
+ */
 void amdgpu_svm_range_restore_begin_compute(struct amdgpu_svm *svm)
 {
 	int ret;
@@ -317,39 +317,6 @@ static int amdgpu_svm_range_lock_vm_pd(struct amdgpu_svm *svm, struct drm_exec *
 	return 0;
 }
 
-static int
-amdgpu_svm_range_update_gpu(struct amdgpu_svm *svm, unsigned long start_page,
-			   unsigned long last_page, uint64_t pte_flags,
-			   dma_addr_t *pages_addr, bool flush_tlb,
-			   bool update_pdes, bool wait_fence)
-{
-	struct drm_exec exec;
-	struct dma_fence *fence = NULL;
-	int ret;
-
-	ret = amdgpu_svm_range_lock_vm_pd(svm, &exec);
-	if (ret)
-		return ret;
-
-	ret = amdgpu_vm_update_range(svm->adev, svm->vm, false, false,
-				     flush_tlb, true,
-				     NULL, start_page, last_page, pte_flags, 0, 0,
-				     NULL, pages_addr, wait_fence ? &fence : NULL);
-	if (!ret && wait_fence && fence) {
-		ret = dma_fence_wait(fence, false);
-		if (ret < 0)
-			AMDGPU_SVM_TRACE("wait unmap fence failed: ret=%d [0x%lx-0x%lx]-0x%lx\n",
-					 ret, start_page, last_page,
-					 last_page - start_page + 1);
-	}
-	if (!ret && update_pdes)
-		ret = amdgpu_vm_update_pdes(svm->adev, svm->vm, false);
-
-	dma_fence_put(fence);
-	drm_exec_fini(&exec);
-	return ret;
-}
-
 static int
 amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
 				  struct drm_gpusvm_range *range,
@@ -376,9 +343,11 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
 						npages - mapped_pages);
 		dma_addr_t seg_addr = entry->addr;
 		unsigned long start_page, last_page;
+		uint64_t seg_pte_flags = pte_flags;
 		bool is_last_seg;
 
-		if (entry->proto != DRM_INTERCONNECT_SYSTEM)
+		if (entry->proto != DRM_INTERCONNECT_SYSTEM &&
+		    entry->proto != AMDGPU_INTERCONNECT_VRAM)
 			return -EOPNOTSUPP;
 
 		while (mapped_pages + seg_pages < npages) {
@@ -399,9 +368,13 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
 		last_page = start_page + seg_pages - 1;
 		is_last_seg = mapped_pages + seg_pages == npages;
 
+		/* For VRAM pages, Clear the SYSTEM and SNOOPED bits */
+		if (entry->proto == AMDGPU_INTERCONNECT_VRAM)
+			seg_pte_flags &= ~(AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED);
+
 		ret = amdgpu_vm_update_range(svm->adev, svm->vm, false, false,
 					     flush_tlb && is_last_seg, true, NULL,
-					     start_page, last_page, pte_flags,
+					     start_page, last_page, seg_pte_flags,
 					     0, seg_addr, NULL, NULL,
 					     wait_fence && is_last_seg ? fence : NULL);
 		if (ret)
@@ -413,17 +386,35 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
 	return 0;
 }
 
+static bool
+preferred_loc_is_vram(const struct amdgpu_svm_attrs *attrs)
+{
+	return attrs->preferred_loc != AMDGPU_SVM_LOCATION_SYSMEM &&
+	       attrs->preferred_loc != AMDGPU_SVM_LOCATION_UNDEFINED;
+}
+
 static int
 amdgpu_svm_range_map(struct amdgpu_svm *svm,
 		       unsigned long start,
 		       unsigned long end,
 		       const struct amdgpu_svm_attrs *attrs,
 		       const struct drm_gpusvm_ctx *gpusvm_ctx,
-		       uint64_t pte_flags)
+		       uint64_t pte_flags,
+		       enum amdgpu_svm_migrate_mode migrate_mode)
 {
 	unsigned long addr = start;
 	int ret;
 
+	/*
+	 * For preferred migration mode, determine the
+	 * actual migration direction based on the preferred
+	 * location attribute.
+	 */
+	if (migrate_mode == AMDGPU_SVM_MIGRATE_PREFERRED)
+		migrate_mode = preferred_loc_is_vram(attrs) ?
+				       AMDGPU_SVM_MIGRATE_TO_VRAM :
+				       AMDGPU_SVM_MIGRATE_TO_SYSMEM;
+
 	while (addr < end) {
 		struct drm_exec exec;
 		struct drm_gpusvm_ctx map_ctx;
@@ -462,6 +453,10 @@ amdgpu_svm_range_map(struct amdgpu_svm *svm,
 		if (next_addr <= addr)
 			return -EINVAL;
 
+		/* Per-range migration */
+		if (migrate_mode != AMDGPU_SVM_MIGRATE_NONE)
+			amdgpu_svm_range_migrate_range(svm, range, migrate_mode);
+
 		range_pte_flags = map_ctx.read_only ?
 			(pte_flags & ~AMDGPU_PTE_WRITEABLE) : pte_flags;
 
@@ -529,10 +524,16 @@ amdgpu_svm_range_map(struct amdgpu_svm *svm,
 static int
 amdgpu_svm_range_map_interval(struct amdgpu_svm *svm, unsigned long start_page,
 				unsigned long last_page,
-				const struct amdgpu_svm_attrs *attrs)
+				const struct amdgpu_svm_attrs *attrs,
+				enum amdgpu_svm_migrate_mode migrate_mode)
 {
+	bool hw_devmem = amdgpu_pagemap_capable(svm);
+
 	struct drm_gpusvm_ctx gpusvm_ctx = {
 		.read_only = !!(attrs->flags & AMDGPU_SVM_FLAG_GPU_RO),
+		.devmem_possible = hw_devmem,
+		.device_private_page_owner = hw_devmem ?
+			AMDGPU_SVM_PGMAP_OWNER(svm->adev) : NULL,
 	};
 	unsigned long start = start_page << PAGE_SHIFT;
 	unsigned long end = (last_page + 1) << PAGE_SHIFT;
@@ -542,7 +543,7 @@ amdgpu_svm_range_map_interval(struct amdgpu_svm *svm, unsigned long start_page,
 	pte_flags = amdgpu_svm_range_attr_pte_flags(svm, attrs);
 
 	ret = amdgpu_svm_range_map(svm, start, end, attrs, &gpusvm_ctx,
-				   pte_flags);
+				   pte_flags, migrate_mode);
 	if (ret)
 		AMDGPU_SVM_TRACE("map_interval failed: ret=%d [0x%lx-0x%lx)-0x%lx\n",
 				 ret, start, end, end - start);
@@ -553,7 +554,8 @@ amdgpu_svm_range_map_interval(struct amdgpu_svm *svm, unsigned long start_page,
 int
 amdgpu_svm_range_map_attr_ranges(struct amdgpu_svm *svm,
 				 unsigned long start_page,
-				 unsigned long last_page)
+				 unsigned long last_page,
+				 enum amdgpu_svm_migrate_mode migrate_mode)
 {
 	lockdep_assert_held_write(&svm->svm_lock);
 
@@ -573,9 +575,10 @@ amdgpu_svm_range_map_attr_ranges(struct amdgpu_svm *svm,
 
 		seg_last = min(seg_last, last_page);
 		if (range_has_access(attrs.access)) {
-			/* map may fail here cause no vma or access deny */
-			ret = amdgpu_svm_range_map_interval(svm, cursor, seg_last,
-							    &attrs);
+			ret = amdgpu_svm_range_map_interval(svm, cursor,
+							    seg_last,
+							    &attrs,
+							    migrate_mode);
 			if (ret)
 				return ret;
 		}
@@ -657,7 +660,6 @@ static int amdgpu_svm_range_rebuild_locked(struct amdgpu_svm *svm,
 	unsigned long rebuild_start = start_page;
 	unsigned long rebuild_last = last_page;
 	bool removed;
-	int ret;
 
 	lockdep_assert_held_write(&svm->svm_lock);
 
@@ -673,14 +675,10 @@ static int amdgpu_svm_range_rebuild_locked(struct amdgpu_svm *svm,
 	/* scan rebuild start end to build the extra removed ranges */
 	if (rebuild)
 		return amdgpu_svm_range_map_attr_ranges(svm, rebuild_start,
-							rebuild_last);
+							rebuild_last,
+							AMDGPU_SVM_MIGRATE_PREFERRED);
 
-	ret = amdgpu_svm_range_update_gpu(svm, rebuild_start, rebuild_last,
-					  0, NULL, true, true, true);
-	if (!ret)
-		svm->flush_tlb(svm);
-
-	return ret;
+	return 0;
 }
 
 static void
@@ -706,10 +704,11 @@ amdgpu_svm_range_process_notifier_ranges(struct amdgpu_svm *svm,
 		if (clear_pte) {
 			amdgpu_svm_range_gpu_unmap_in_notifier(svm, range,
 									   mmu_range);
-			range_invalidate_gpu_mapping(range);
 		}
 
 		drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx);
+		range_invalidate_gpu_mapping(range);
+
 		if (is_unmap)
 			drm_gpusvm_range_set_unmapped(range, mmu_range);
 
@@ -758,6 +757,7 @@ int amdgpu_svm_range_apply_attr_change(struct amdgpu_svm *svm,
 {
 	lockdep_assert_held_write(&svm->svm_lock);
 
+	enum amdgpu_svm_migrate_mode migrate_mode = AMDGPU_SVM_MIGRATE_NONE;
 	bool old_access, new_access;
 	bool update_mapping = false;
 
@@ -788,16 +788,24 @@ int amdgpu_svm_range_apply_attr_change(struct amdgpu_svm *svm,
 	    new_access)
 		update_mapping = true;
 
-	if (trigger & AMDGPU_SVM_ATTR_TRIGGER_LOCATION_CHANGE) {
-		/* TODO: add migration */
+	if ((trigger & AMDGPU_SVM_ATTR_TRIGGER_LOCATION_CHANGE) &&
+	    new_access) {
+		int32_t loc = new_attrs->prefetch_loc;
+
+		if (loc == AMDGPU_SVM_LOCATION_SYSMEM) {
+			migrate_mode = AMDGPU_SVM_MIGRATE_TO_SYSMEM;
+			update_mapping = true;
+		} else if (loc != AMDGPU_SVM_LOCATION_UNDEFINED) {
+			migrate_mode = AMDGPU_SVM_MIGRATE_TO_VRAM;
+			update_mapping = true;
+		}
 	}
 
 	if (!update_mapping)
 		return 0;
 
-	AMDGPU_SVM_TRACE("mapping update: remap interval [0x%lx-0x%lx]-0x%lx\n",
-			 start, last, last - start + 1);
-	return amdgpu_svm_range_map_interval(svm, start, last, new_attrs);
+	return amdgpu_svm_range_map_interval(svm, start, last, new_attrs,
+					     migrate_mode);
 }
 
 static bool
@@ -1004,7 +1012,8 @@ static void amdgpu_svm_range_restore_worker(struct work_struct *w)
 
 		down_write(&svm->svm_lock);
 		ret = amdgpu_svm_range_map_attr_ranges(svm, op_ctx.start,
-						       op_ctx.last);
+						       op_ctx.last,
+						       AMDGPU_SVM_MIGRATE_NONE);
 		up_write(&svm->svm_lock);
 
 		if (ret) {
@@ -1042,7 +1051,6 @@ static void amdgpu_svm_range_restore_worker(struct work_struct *w)
 			drm_gpusvm_notifier_unlock(&svm->gpusvm);
 			svm->end_restore(svm);
 			return;
-	
 		}
 		drm_gpusvm_notifier_unlock(&svm->gpusvm);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
index 18bf3dad13fd..0065ae50c700 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
@@ -30,6 +30,8 @@
 #include <linux/list.h>
 #include <linux/types.h>
 
+#include "amdgpu_svm_range_migrate.h"
+
 struct amdgpu_svm;
 struct amdgpu_svm_attrs;
 struct drm_gpusvm_notifier;
@@ -62,7 +64,8 @@ void amdgpu_svm_range_flush(struct amdgpu_svm *svm);
 void amdgpu_svm_range_sync_work(struct amdgpu_svm *svm);
 int amdgpu_svm_range_map_attr_ranges(struct amdgpu_svm *svm,
 				     unsigned long start_page,
-				     unsigned long last_page);
+				     unsigned long last_page,
+				     enum amdgpu_svm_migrate_mode migrate_mode);
 int amdgpu_svm_range_apply_attr_change(
 	struct amdgpu_svm *svm, unsigned long start, unsigned long last,
 	uint32_t trigger, const struct amdgpu_svm_attrs *prev_attrs,
-- 
2.34.1