system/ram-discard-manager.c - qemu - Git at Google

 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
  * RAM Discard Manager
  *
  * Copyright Red Hat, Inc. 2026
  */

 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "qemu/queue.h"
 #include "system/memory.h"

 static uint64_t ram_discard_source_get_min_granularity(const RamDiscardSource *rds,
                                                        const MemoryRegion *mr)
 {
     RamDiscardSourceClass *rdsc = RAM_DISCARD_SOURCE_GET_CLASS(rds);

     g_assert(rdsc->get_min_granularity);
     return rdsc->get_min_granularity(rds, mr);
 }

 static bool ram_discard_source_is_populated(const RamDiscardSource *rds,
                                             const MemoryRegionSection *section)
 {
     RamDiscardSourceClass *rdsc = RAM_DISCARD_SOURCE_GET_CLASS(rds);

     g_assert(rdsc->is_populated);
     return rdsc->is_populated(rds, section);
 }

 /*
  * Iterate a single source's populated or discarded regions and call
  * replay_fn for each contiguous run.
  */
 static int replay_source_by_state(const RamDiscardSource *source,
                                   const MemoryRegion *mr,
                                   const MemoryRegionSection *section,
                                   bool replay_populated,
                                   ReplayRamDiscardState replay_fn,
                                   void *opaque)
 {
     uint64_t granularity, offset, size, end, pos, run_start = 0;
     bool in_run = false;
     int ret = 0;

     granularity = ram_discard_source_get_min_granularity(source, mr);
     offset = section->offset_within_region;
     size = int128_get64(section->size);
     end = offset + size;

     /* Align iteration to granularity boundaries */
     pos = QEMU_ALIGN_DOWN(offset, granularity);

     for (; pos < end; pos += granularity) {
         MemoryRegionSection chunk = {
             .mr = section->mr,
             .offset_within_region = pos,
             .size = int128_make64(granularity),
         };
         bool populated = ram_discard_source_is_populated(source, &chunk);

         if (populated == replay_populated) {
             if (!in_run) {
                 run_start = pos;
                 in_run = true;
             }
         } else if (in_run) {
             MemoryRegionSection tmp = *section;

             if (memory_region_section_intersect_range(&tmp, run_start,
                                                       pos - run_start)) {
                 ret = replay_fn(&tmp, opaque);
                 if (ret) {
                     return ret;
                 }
             }
             in_run = false;
         }
     }

     if (in_run) {
         MemoryRegionSection tmp = *section;

         if (memory_region_section_intersect_range(&tmp, run_start,
                                                   pos - run_start)) {
             ret = replay_fn(&tmp, opaque);
         }
     }

     return ret;
 }

 RamDiscardManager *ram_discard_manager_new(MemoryRegion *mr)
 {
     RamDiscardManager *rdm;

     rdm = RAM_DISCARD_MANAGER(object_new(TYPE_RAM_DISCARD_MANAGER));
     rdm->mr = mr;
     return rdm;
 }

 static void ram_discard_manager_update_granularity(RamDiscardManager *rdm)
 {
     RamDiscardSourceEntry *entry;
     uint64_t granularity = 0;

     QLIST_FOREACH(entry, &rdm->source_list, next) {
         uint64_t src_granularity;

         src_granularity =
             ram_discard_source_get_min_granularity(entry->rds, rdm->mr);
         g_assert(src_granularity != 0);
         if (granularity == 0) {
             granularity = src_granularity;
         } else {
             granularity = MIN(granularity, src_granularity);
         }
     }
     rdm->min_granularity = granularity;
 }

 static RamDiscardSourceEntry *
 ram_discard_manager_find_source(RamDiscardManager *rdm, RamDiscardSource *rds)
 {
     RamDiscardSourceEntry *entry;

     QLIST_FOREACH(entry, &rdm->source_list, next) {
         if (entry->rds == rds) {
             return entry;
         }
     }
     return NULL;
 }

 static int rdl_populate_cb(const MemoryRegionSection *section, void *opaque)
 {
     RamDiscardListener *rdl = opaque;
     MemoryRegionSection tmp = *rdl->section;

     g_assert(section->mr == rdl->section->mr);

     if (!memory_region_section_intersect_range(&tmp,
                                                section->offset_within_region,
                                                int128_get64(section->size))) {
         return 0;
     }

     return rdl->notify_populate(rdl, &tmp);
 }

 static int rdl_discard_cb(const MemoryRegionSection *section, void *opaque)
 {
     RamDiscardListener *rdl = opaque;
     MemoryRegionSection tmp = *rdl->section;

     g_assert(section->mr == rdl->section->mr);

     if (!memory_region_section_intersect_range(&tmp,
                                                section->offset_within_region,
                                                int128_get64(section->size))) {
         return 0;
     }

     rdl->notify_discard(rdl, &tmp);
     return 0;
 }

 static bool rdm_is_all_populated_skip(const RamDiscardManager *rdm,
                                       const MemoryRegionSection *section,
                                       const RamDiscardSource *skip_source)
 {
     RamDiscardSourceEntry *entry;

     QLIST_FOREACH(entry, &rdm->source_list, next) {
         if (skip_source && entry->rds == skip_source) {
             continue;
         }
         if (!ram_discard_source_is_populated(entry->rds, section)) {
             return false;
         }
     }
     return true;
 }

 typedef struct SourceNotifyCtx {
     RamDiscardManager *rdm;
     RamDiscardListener *rdl;
     RamDiscardSource *source; /* added or removed */
 } SourceNotifyCtx;

 /*
  * Unified helper to replay regions based on populated state.
  * If replay_populated is true: replay regions where ALL sources are populated.
  * If replay_populated is false: replay regions where ANY source is discarded.
  */
 static int replay_by_populated_state(const RamDiscardManager *rdm,
                                      const MemoryRegionSection *section,
                                      const RamDiscardSource *skip_source,
                                      bool replay_populated,
                                      ReplayRamDiscardState replay_fn,
                                      void *user_opaque)
 {
     uint64_t granularity = rdm->min_granularity;
     uint64_t offset, end_offset;
     uint64_t run_start = 0;
     bool in_run = false;
     int ret = 0;

     if (QLIST_EMPTY(&rdm->source_list)) {
         if (replay_populated) {
             return replay_fn(section, user_opaque);
         }
         return 0;
     }

     g_assert(granularity != 0);

     offset = section->offset_within_region;
     end_offset = offset + int128_get64(section->size);

     while (offset < end_offset) {
         MemoryRegionSection subsection = {
             .mr = section->mr,
             .offset_within_region = offset,
             .size = int128_make64(MIN(granularity, end_offset - offset)),
         };
         bool all_populated;
         bool included;

         all_populated = rdm_is_all_populated_skip(rdm, &subsection,
                                                      skip_source);
         included = replay_populated ? all_populated : !all_populated;

         if (included) {
             if (!in_run) {
                 run_start = offset;
                 in_run = true;
             }
         } else {
             if (in_run) {
                 MemoryRegionSection run_section = {
                     .mr = section->mr,
                     .offset_within_region = run_start,
                     .size = int128_make64(offset - run_start),
                 };
                 ret = replay_fn(&run_section, user_opaque);
                 if (ret) {
                     return ret;
                 }
                 in_run = false;
             }
         }
         if (granularity > end_offset - offset) {
             break;
         }
         offset += granularity;
     }

     if (in_run) {
         MemoryRegionSection run_section = {
             .mr = section->mr,
             .offset_within_region = run_start,
             .size = int128_make64(end_offset - run_start),
         };
         ret = replay_fn(&run_section, user_opaque);
     }

     return ret;
 }

 static int add_source_check_discard_cb(const MemoryRegionSection *section,
                                        void *opaque)
 {
     SourceNotifyCtx *ctx = opaque;

     return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
                                      rdl_discard_cb, ctx->rdl);
 }

 static int del_source_check_populate_cb(const MemoryRegionSection *section,
                                         void *opaque)
 {
     SourceNotifyCtx *ctx = opaque;

     return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
                                      rdl_populate_cb, ctx->rdl);
 }

 int ram_discard_manager_add_source(RamDiscardManager *rdm,
                                    RamDiscardSource *source)
 {
     RamDiscardSourceEntry *entry;
     RamDiscardListener *rdl, *rdl2;
     int ret = 0;

     if (ram_discard_manager_find_source(rdm, source)) {
         return -EBUSY;
     }

     /*
      * If there are existing listeners, notify them about regions that
      * become discarded due to adding this source. Only notify for regions
      * that were previously populated (all other sources agreed).
      */
     QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
         SourceNotifyCtx ctx = {
             .rdm = rdm,
             .rdl = rdl,
             /* no need to set source */
         };
         ret = replay_source_by_state(source, rdm->mr, rdl->section,
                                      false,
                                      add_source_check_discard_cb, &ctx);
         if (ret) {
             break;
         }
     }
     if (ret) {
         QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
             SourceNotifyCtx ctx = {
                 .rdm = rdm,
                 .rdl = rdl2,
             };
             replay_source_by_state(source, rdm->mr, rdl2->section,
                                    false,
                                    del_source_check_populate_cb,
                                    &ctx);
             if (rdl == rdl2) {
                 break;
             }
         }

         return ret;
     }

     entry = g_new0(RamDiscardSourceEntry, 1);
     entry->rds = source;
     QLIST_INSERT_HEAD(&rdm->source_list, entry, next);

     ram_discard_manager_update_granularity(rdm);

     return ret;
 }

 int ram_discard_manager_del_source(RamDiscardManager *rdm,
                                    RamDiscardSource *source)
 {
     RamDiscardSourceEntry *entry;
     RamDiscardListener *rdl, *rdl2;
     int ret = 0;

     entry = ram_discard_manager_find_source(rdm, source);
     if (!entry) {
         return -ENOENT;
     }

     /*
      * If there are existing listeners, check if any regions become
      * populated due to removing this source.
      */
     QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
         SourceNotifyCtx ctx = {
             .rdm = rdm,
             .rdl = rdl,
             .source = source,
         };
         /*
          * From the previously discarded regions, check if any
          * regions become populated.
          */
         ret = replay_source_by_state(source, rdm->mr, rdl->section,
                                      false,
                                      del_source_check_populate_cb,
                                      &ctx);
         if (ret) {
             break;
         }
     }
     if (ret) {
         QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
             SourceNotifyCtx ctx = {
                 .rdm = rdm,
                 .rdl = rdl2,
                 .source = source,
             };
             replay_source_by_state(source, rdm->mr, rdl2->section,
                                    false,
                                    add_source_check_discard_cb,
                                    &ctx);
             if (rdl == rdl2) {
                 break;
             }
         }

         return ret;
     }

     QLIST_REMOVE(entry, next);
     g_free(entry);
     ram_discard_manager_update_granularity(rdm);
     return ret;
 }

 uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm,
                                                  const MemoryRegion *mr)
 {
     g_assert(mr == rdm->mr);
     return rdm->min_granularity;
 }

 /*
  * Aggregated query: returns true only if ALL sources report populated (AND).
  */
 bool ram_discard_manager_is_populated(const RamDiscardManager *rdm,
                                       const MemoryRegionSection *section)
 {
     RamDiscardSourceEntry *entry;

     QLIST_FOREACH(entry, &rdm->source_list, next) {
         if (!ram_discard_source_is_populated(entry->rds, section)) {
             return false;
         }
     }
     return true;
 }

 int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
                                          const MemoryRegionSection *section,
                                          ReplayRamDiscardState replay_fn,
                                          void *opaque)
 {
     return replay_by_populated_state(rdm, section, NULL, true,
                                      replay_fn, opaque);
 }

 int ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
                                          const MemoryRegionSection *section,
                                          ReplayRamDiscardState replay_fn,
                                          void *opaque)
 {
     return replay_by_populated_state(rdm, section, NULL, false,
                                      replay_fn, opaque);
 }

 static void ram_discard_manager_initfn(Object *obj)
 {
     RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);

     QLIST_INIT(&rdm->source_list);
     QLIST_INIT(&rdm->rdl_list);
     rdm->min_granularity = 0;
 }

 static void ram_discard_manager_finalize(Object *obj)
 {
     RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);

     g_assert(QLIST_EMPTY(&rdm->rdl_list));
     g_assert(QLIST_EMPTY(&rdm->source_list));
 }

 int ram_discard_manager_notify_populate(RamDiscardManager *rdm,
                                         RamDiscardSource *source,
                                         uint64_t offset, uint64_t size)
 {
     RamDiscardListener *rdl, *rdl2;
     MemoryRegionSection section = {
         .mr = rdm->mr,
         .offset_within_region = offset,
         .size = int128_make64(size),
     };
     int ret = 0;

     g_assert(ram_discard_manager_find_source(rdm, source));

     /*
      * Only notify about regions that are populated in ALL sources.
      * Skip the calling source: it has implicitly declared itself populated
      * for this range but may not have updated its bitmap yet.
      */
     QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
         ret = replay_by_populated_state(rdm, &section, source, true,
                                         rdl_populate_cb, rdl);
         if (ret) {
             break;
         }
     }

     if (ret) {
         /*
          * Rollback: notify discard for listeners we already notified,
          * including the failing listener which may have been partially
          * notified. Listeners must handle discard notifications for
          * regions they didn't receive populate notifications for.
          */
         QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
             replay_by_populated_state(rdm, &section, source, true,
                                       rdl_discard_cb, rdl2);
             if (rdl2 == rdl) {
                 break;
             }
         }
     }
     return ret;
 }

 void ram_discard_manager_notify_discard(RamDiscardManager *rdm,
                                         RamDiscardSource *source,
                                         uint64_t offset, uint64_t size)
 {
     RamDiscardListener *rdl;
     MemoryRegionSection section = {
         .mr = rdm->mr,
         .offset_within_region = offset,
         .size = int128_make64(size),
     };

     g_assert(ram_discard_manager_find_source(rdm, source));

     /*
      * Only notify about ranges that were aggregately populated before this
      * source's discard. Since the source has already updated its state,
      * we use replay_by_populated_state with this source skipped - it will
      * replay only the ranges where all OTHER sources are populated.
      */
     QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
         replay_by_populated_state(rdm, &section, source, true,
                                   rdl_discard_cb, rdl);
     }
 }

 void ram_discard_manager_notify_discard_all(RamDiscardManager *rdm,
                                             RamDiscardSource *source)
 {
     RamDiscardListener *rdl;

     g_assert(ram_discard_manager_find_source(rdm, source));

     QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
         rdl->notify_discard(rdl, rdl->section);
     }
 }

 void ram_discard_manager_register_listener(RamDiscardManager *rdm,
                                            RamDiscardListener *rdl,
                                            MemoryRegionSection *section)
 {
     int ret;

     g_assert(section->mr == rdm->mr);

     object_ref(rdm);
     rdl->section = memory_region_section_new_copy(section);
     QLIST_INSERT_HEAD(&rdm->rdl_list, rdl, next);

     ret = ram_discard_manager_replay_populated(rdm, rdl->section,
                                                rdl_populate_cb, rdl);
     if (ret) {
         error_report("%s: Replaying populated ranges failed: %s", __func__,
                      strerror(-ret));
     }
 }

 void ram_discard_manager_unregister_listener(RamDiscardManager *rdm,
                                              RamDiscardListener *rdl)
 {
     g_assert(rdl->section);
     g_assert(rdl->section->mr == rdm->mr);

     rdl->notify_discard(rdl, rdl->section);
     memory_region_section_free_copy(rdl->section);
     rdl->section = NULL;
     QLIST_REMOVE(rdl, next);
     object_unref(rdm);
 }

 int ram_discard_manager_replay_populated_to_listeners(RamDiscardManager *rdm)
 {
     RamDiscardListener *rdl;
     int ret = 0;

     QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
         ret = ram_discard_manager_replay_populated(rdm, rdl->section,
                                                    rdl_populate_cb, rdl);
         if (ret) {
             break;
         }
     }
     return ret;
 }

 static const TypeInfo ram_discard_manager_info = {
     .parent             = TYPE_OBJECT,
     .name               = TYPE_RAM_DISCARD_MANAGER,
     .instance_size      = sizeof(RamDiscardManager),
     .instance_init      = ram_discard_manager_initfn,
     .instance_finalize  = ram_discard_manager_finalize,
 };

 static const TypeInfo ram_discard_source_info = {
     .parent             = TYPE_INTERFACE,
     .name               = TYPE_RAM_DISCARD_SOURCE,
     .class_size         = sizeof(RamDiscardSourceClass),
 };

 static void ram_discard_manager_register_types(void)
 {
     type_register_static(&ram_discard_manager_info);
     type_register_static(&ram_discard_source_info);
 }

 type_init(ram_discard_manager_register_types)
	/* SPDX-License-Identifier: GPL-2.0-or-later */
	/*
	* RAM Discard Manager
	*
	* Copyright Red Hat, Inc. 2026
	*/

	#include "qemu/osdep.h"
	#include "qemu/error-report.h"
	#include "qemu/queue.h"
	#include "system/memory.h"

	static uint64_t ram_discard_source_get_min_granularity(const RamDiscardSource *rds,
	const MemoryRegion *mr)
	{
	RamDiscardSourceClass *rdsc = RAM_DISCARD_SOURCE_GET_CLASS(rds);

	g_assert(rdsc->get_min_granularity);
	return rdsc->get_min_granularity(rds, mr);
	}

	static bool ram_discard_source_is_populated(const RamDiscardSource *rds,
	const MemoryRegionSection *section)
	{
	RamDiscardSourceClass *rdsc = RAM_DISCARD_SOURCE_GET_CLASS(rds);

	g_assert(rdsc->is_populated);
	return rdsc->is_populated(rds, section);
	}

	/*
	* Iterate a single source's populated or discarded regions and call
	* replay_fn for each contiguous run.
	*/
	static int replay_source_by_state(const RamDiscardSource *source,
	const MemoryRegion *mr,
	const MemoryRegionSection *section,
	bool replay_populated,
	ReplayRamDiscardState replay_fn,
	void *opaque)
	{
	uint64_t granularity, offset, size, end, pos, run_start = 0;
	bool in_run = false;
	int ret = 0;

	granularity = ram_discard_source_get_min_granularity(source, mr);
	offset = section->offset_within_region;
	size = int128_get64(section->size);
	end = offset + size;

	/* Align iteration to granularity boundaries */
	pos = QEMU_ALIGN_DOWN(offset, granularity);

	for (; pos < end; pos += granularity) {
	MemoryRegionSection chunk = {
	.mr = section->mr,
	.offset_within_region = pos,
	.size = int128_make64(granularity),
	};
	bool populated = ram_discard_source_is_populated(source, &chunk);

	if (populated == replay_populated) {
	if (!in_run) {
	run_start = pos;
	in_run = true;
	}
	} else if (in_run) {
	MemoryRegionSection tmp = *section;

	if (memory_region_section_intersect_range(&tmp, run_start,
	pos - run_start)) {
	ret = replay_fn(&tmp, opaque);
	if (ret) {
	return ret;
	}
	}
	in_run = false;
	}
	}

	if (in_run) {
	MemoryRegionSection tmp = *section;

	if (memory_region_section_intersect_range(&tmp, run_start,
	pos - run_start)) {
	ret = replay_fn(&tmp, opaque);
	}
	}

	return ret;
	}

	RamDiscardManager ram_discard_manager_new(MemoryRegion mr)
	{
	RamDiscardManager *rdm;

	rdm = RAM_DISCARD_MANAGER(object_new(TYPE_RAM_DISCARD_MANAGER));
	rdm->mr = mr;
	return rdm;
	}

	static void ram_discard_manager_update_granularity(RamDiscardManager *rdm)
	{
	RamDiscardSourceEntry *entry;
	uint64_t granularity = 0;

	QLIST_FOREACH(entry, &rdm->source_list, next) {
	uint64_t src_granularity;

	src_granularity =
	ram_discard_source_get_min_granularity(entry->rds, rdm->mr);
	g_assert(src_granularity != 0);
	if (granularity == 0) {
	granularity = src_granularity;
	} else {
	granularity = MIN(granularity, src_granularity);
	}
	}
	rdm->min_granularity = granularity;
	}

	static RamDiscardSourceEntry *
	ram_discard_manager_find_source(RamDiscardManager rdm, RamDiscardSource rds)
	{
	RamDiscardSourceEntry *entry;

	QLIST_FOREACH(entry, &rdm->source_list, next) {
	if (entry->rds == rds) {
	return entry;
	}
	}
	return NULL;
	}

	static int rdl_populate_cb(const MemoryRegionSection section, void opaque)
	{
	RamDiscardListener *rdl = opaque;
	MemoryRegionSection tmp = *rdl->section;

	g_assert(section->mr == rdl->section->mr);

	if (!memory_region_section_intersect_range(&tmp,
	section->offset_within_region,
	int128_get64(section->size))) {
	return 0;
	}

	return rdl->notify_populate(rdl, &tmp);
	}

	static int rdl_discard_cb(const MemoryRegionSection section, void opaque)
	{
	RamDiscardListener *rdl = opaque;
	MemoryRegionSection tmp = *rdl->section;

	g_assert(section->mr == rdl->section->mr);

	if (!memory_region_section_intersect_range(&tmp,
	section->offset_within_region,
	int128_get64(section->size))) {
	return 0;
	}

	rdl->notify_discard(rdl, &tmp);
	return 0;
	}

	static bool rdm_is_all_populated_skip(const RamDiscardManager *rdm,
	const MemoryRegionSection *section,
	const RamDiscardSource *skip_source)
	{
	RamDiscardSourceEntry *entry;

	QLIST_FOREACH(entry, &rdm->source_list, next) {
	if (skip_source && entry->rds == skip_source) {
	continue;
	}
	if (!ram_discard_source_is_populated(entry->rds, section)) {
	return false;
	}
	}
	return true;
	}

	typedef struct SourceNotifyCtx {
	RamDiscardManager *rdm;
	RamDiscardListener *rdl;
	RamDiscardSource source; / added or removed */
	} SourceNotifyCtx;

	/*
	* Unified helper to replay regions based on populated state.
	* If replay_populated is true: replay regions where ALL sources are populated.
	* If replay_populated is false: replay regions where ANY source is discarded.
	*/
	static int replay_by_populated_state(const RamDiscardManager *rdm,
	const MemoryRegionSection *section,
	const RamDiscardSource *skip_source,
	bool replay_populated,
	ReplayRamDiscardState replay_fn,
	void *user_opaque)
	{
	uint64_t granularity = rdm->min_granularity;
	uint64_t offset, end_offset;
	uint64_t run_start = 0;
	bool in_run = false;
	int ret = 0;

	if (QLIST_EMPTY(&rdm->source_list)) {
	if (replay_populated) {
	return replay_fn(section, user_opaque);
	}
	return 0;
	}

	g_assert(granularity != 0);

	offset = section->offset_within_region;
	end_offset = offset + int128_get64(section->size);

	while (offset < end_offset) {
	MemoryRegionSection subsection = {
	.mr = section->mr,
	.offset_within_region = offset,
	.size = int128_make64(MIN(granularity, end_offset - offset)),
	};
	bool all_populated;
	bool included;

	all_populated = rdm_is_all_populated_skip(rdm, &subsection,
	skip_source);
	included = replay_populated ? all_populated : !all_populated;

	if (included) {
	if (!in_run) {
	run_start = offset;
	in_run = true;
	}
	} else {
	if (in_run) {
	MemoryRegionSection run_section = {
	.mr = section->mr,
	.offset_within_region = run_start,
	.size = int128_make64(offset - run_start),
	};
	ret = replay_fn(&run_section, user_opaque);
	if (ret) {
	return ret;
	}
	in_run = false;
	}
	}
	if (granularity > end_offset - offset) {
	break;
	}
	offset += granularity;
	}

	if (in_run) {
	MemoryRegionSection run_section = {
	.mr = section->mr,
	.offset_within_region = run_start,
	.size = int128_make64(end_offset - run_start),
	};
	ret = replay_fn(&run_section, user_opaque);
	}

	return ret;
	}

	static int add_source_check_discard_cb(const MemoryRegionSection *section,
	void *opaque)
	{
	SourceNotifyCtx *ctx = opaque;

	return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
	rdl_discard_cb, ctx->rdl);
	}

	static int del_source_check_populate_cb(const MemoryRegionSection *section,
	void *opaque)
	{
	SourceNotifyCtx *ctx = opaque;

	return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
	rdl_populate_cb, ctx->rdl);
	}

	int ram_discard_manager_add_source(RamDiscardManager *rdm,
	RamDiscardSource *source)
	{
	RamDiscardSourceEntry *entry;
	RamDiscardListener rdl, rdl2;
	int ret = 0;

	if (ram_discard_manager_find_source(rdm, source)) {
	return -EBUSY;
	}

	/*
	* If there are existing listeners, notify them about regions that
	* become discarded due to adding this source. Only notify for regions
	* that were previously populated (all other sources agreed).
	*/
	QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
	SourceNotifyCtx ctx = {
	.rdm = rdm,
	.rdl = rdl,
	/* no need to set source */
	};
	ret = replay_source_by_state(source, rdm->mr, rdl->section,
	false,
	add_source_check_discard_cb, &ctx);
	if (ret) {
	break;
	}
	}
	if (ret) {
	QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
	SourceNotifyCtx ctx = {
	.rdm = rdm,
	.rdl = rdl2,
	};
	replay_source_by_state(source, rdm->mr, rdl2->section,
	false,
	del_source_check_populate_cb,
	&ctx);
	if (rdl == rdl2) {
	break;
	}
	}

	return ret;
	}

	entry = g_new0(RamDiscardSourceEntry, 1);
	entry->rds = source;
	QLIST_INSERT_HEAD(&rdm->source_list, entry, next);

	ram_discard_manager_update_granularity(rdm);

	return ret;
	}

	int ram_discard_manager_del_source(RamDiscardManager *rdm,
	RamDiscardSource *source)
	{
	RamDiscardSourceEntry *entry;
	RamDiscardListener rdl, rdl2;
	int ret = 0;

	entry = ram_discard_manager_find_source(rdm, source);
	if (!entry) {
	return -ENOENT;
	}

	/*
	* If there are existing listeners, check if any regions become
	* populated due to removing this source.
	*/
	QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
	SourceNotifyCtx ctx = {
	.rdm = rdm,
	.rdl = rdl,
	.source = source,
	};
	/*
	* From the previously discarded regions, check if any
	* regions become populated.
	*/
	ret = replay_source_by_state(source, rdm->mr, rdl->section,
	false,
	del_source_check_populate_cb,
	&ctx);
	if (ret) {
	break;
	}
	}
	if (ret) {
	QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
	SourceNotifyCtx ctx = {
	.rdm = rdm,
	.rdl = rdl2,
	.source = source,
	};
	replay_source_by_state(source, rdm->mr, rdl2->section,
	false,
	add_source_check_discard_cb,
	&ctx);
	if (rdl == rdl2) {
	break;
	}
	}

	return ret;
	}

	QLIST_REMOVE(entry, next);
	g_free(entry);
	ram_discard_manager_update_granularity(rdm);
	return ret;
	}

	uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm,
	const MemoryRegion *mr)
	{
	g_assert(mr == rdm->mr);
	return rdm->min_granularity;
	}

	/*
	* Aggregated query: returns true only if ALL sources report populated (AND).
	*/
	bool ram_discard_manager_is_populated(const RamDiscardManager *rdm,
	const MemoryRegionSection *section)
	{
	RamDiscardSourceEntry *entry;

	QLIST_FOREACH(entry, &rdm->source_list, next) {
	if (!ram_discard_source_is_populated(entry->rds, section)) {
	return false;
	}
	}
	return true;
	}

	int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
	const MemoryRegionSection *section,
	ReplayRamDiscardState replay_fn,
	void *opaque)
	{
	return replay_by_populated_state(rdm, section, NULL, true,
	replay_fn, opaque);
	}

	int ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
	const MemoryRegionSection *section,
	ReplayRamDiscardState replay_fn,
	void *opaque)
	{
	return replay_by_populated_state(rdm, section, NULL, false,
	replay_fn, opaque);
	}

	static void ram_discard_manager_initfn(Object *obj)
	{
	RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);

	QLIST_INIT(&rdm->source_list);
	QLIST_INIT(&rdm->rdl_list);
	rdm->min_granularity = 0;
	}

	static void ram_discard_manager_finalize(Object *obj)
	{
	RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);

	g_assert(QLIST_EMPTY(&rdm->rdl_list));
	g_assert(QLIST_EMPTY(&rdm->source_list));
	}

	int ram_discard_manager_notify_populate(RamDiscardManager *rdm,
	RamDiscardSource *source,
	uint64_t offset, uint64_t size)
	{
	RamDiscardListener rdl, rdl2;
	MemoryRegionSection section = {
	.mr = rdm->mr,
	.offset_within_region = offset,
	.size = int128_make64(size),
	};
	int ret = 0;

	g_assert(ram_discard_manager_find_source(rdm, source));

	/*
	* Only notify about regions that are populated in ALL sources.
	* Skip the calling source: it has implicitly declared itself populated
	* for this range but may not have updated its bitmap yet.
	*/
	QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
	ret = replay_by_populated_state(rdm, &section, source, true,
	rdl_populate_cb, rdl);
	if (ret) {
	break;
	}
	}

	if (ret) {
	/*
	* Rollback: notify discard for listeners we already notified,
	* including the failing listener which may have been partially
	* notified. Listeners must handle discard notifications for
	* regions they didn't receive populate notifications for.
	*/
	QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
	replay_by_populated_state(rdm, &section, source, true,
	rdl_discard_cb, rdl2);
	if (rdl2 == rdl) {
	break;
	}
	}
	}
	return ret;
	}

	void ram_discard_manager_notify_discard(RamDiscardManager *rdm,
	RamDiscardSource *source,
	uint64_t offset, uint64_t size)
	{
	RamDiscardListener *rdl;
	MemoryRegionSection section = {
	.mr = rdm->mr,
	.offset_within_region = offset,
	.size = int128_make64(size),
	};

	g_assert(ram_discard_manager_find_source(rdm, source));

	/*
	* Only notify about ranges that were aggregately populated before this
	* source's discard. Since the source has already updated its state,
	* we use replay_by_populated_state with this source skipped - it will
	* replay only the ranges where all OTHER sources are populated.
	*/
	QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
	replay_by_populated_state(rdm, &section, source, true,
	rdl_discard_cb, rdl);
	}
	}

	void ram_discard_manager_notify_discard_all(RamDiscardManager *rdm,
	RamDiscardSource *source)
	{
	RamDiscardListener *rdl;

	g_assert(ram_discard_manager_find_source(rdm, source));

	QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
	rdl->notify_discard(rdl, rdl->section);
	}
	}

	void ram_discard_manager_register_listener(RamDiscardManager *rdm,
	RamDiscardListener *rdl,
	MemoryRegionSection *section)
	{
	int ret;

	g_assert(section->mr == rdm->mr);

	object_ref(rdm);
	rdl->section = memory_region_section_new_copy(section);
	QLIST_INSERT_HEAD(&rdm->rdl_list, rdl, next);

	ret = ram_discard_manager_replay_populated(rdm, rdl->section,
	rdl_populate_cb, rdl);
	if (ret) {
	error_report("%s: Replaying populated ranges failed: %s", __func__,
	strerror(-ret));
	}
	}

	void ram_discard_manager_unregister_listener(RamDiscardManager *rdm,
	RamDiscardListener *rdl)
	{
	g_assert(rdl->section);
	g_assert(rdl->section->mr == rdm->mr);

	rdl->notify_discard(rdl, rdl->section);
	memory_region_section_free_copy(rdl->section);
	rdl->section = NULL;
	QLIST_REMOVE(rdl, next);
	object_unref(rdm);
	}

	int ram_discard_manager_replay_populated_to_listeners(RamDiscardManager *rdm)
	{
	RamDiscardListener *rdl;
	int ret = 0;

	QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
	ret = ram_discard_manager_replay_populated(rdm, rdl->section,
	rdl_populate_cb, rdl);
	if (ret) {
	break;
	}
	}
	return ret;
	}

	static const TypeInfo ram_discard_manager_info = {
	.parent = TYPE_OBJECT,
	.name = TYPE_RAM_DISCARD_MANAGER,
	.instance_size = sizeof(RamDiscardManager),
	.instance_init = ram_discard_manager_initfn,
	.instance_finalize = ram_discard_manager_finalize,
	};

	static const TypeInfo ram_discard_source_info = {
	.parent = TYPE_INTERFACE,
	.name = TYPE_RAM_DISCARD_SOURCE,
	.class_size = sizeof(RamDiscardSourceClass),
	};

	static void ram_discard_manager_register_types(void)
	{
	type_register_static(&ram_discard_manager_info);
	type_register_static(&ram_discard_source_info);
	}

	type_init(ram_discard_manager_register_types)