util/mesa-db: Introduce multipart mesa-db cache

Whenever a single file mesa-db cache hits max size limit, a half of cache
is evicted and the cache file is defragmented. The downside of this eviction
strategy is that it causes high disk IO usage during eviction if mesa-db
cache file size is large.

In order to mitigate this downside, we will split mesa-db into multiple
part such that only one part will be evicted at a time. Each part will be
an individual single file mesa-db cache, like a DB shard. The new multipart
mesa-db cache will merge the parts into a single virtual cache.

This patch introduces two new environment variables:

1. MESA_DISK_CACHE_DATABASE_NUM_PARTS:
Controls number of mesa-db cache file parts. By default 50 parts will be
created. The old pre-multipart mesa-db cache files will be auto-removed
if they exist, i.e. Mesa will switch to the new DB version automatically.

2. MESA_DISK_CACHE_DATABASE_EVICTION_SCORE_2X_PERIOD:
Controls the eviction score doubling time period. The evicted DB part
selection is based on cache entries size weighted by 'last_access_time' of
the entries. By default the cache eviction score is doubled for each month
of cache entry age, i.e. for two equally sized entries where one entry is
older by one month than the other, the older entry will have x2 eviction
score than the other entry. Database part with a highest total eviction
score is selected for eviction.

This patch brings x40 performance improvement of cache eviction time using
multipart cache vs a single file cache due to a smaller eviction portions
and more optimized eviction algorithm.

Acked-by: Timothy Arceri <tarceri@itsqueeze.com>
Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20256>

docs/envvars.rst

@@ -251,6 +251,17 @@ Core Mesa environment variables
    or else within ``.cache/mesa_shader_cache_db`` within the user's home
    directory.
 
+.. envvar:: MESA_DISK_CACHE_DATABASE_NUM_PARTS
+
+   specifies number of mesa-db cache parts, default is 50.
+
+.. envvar:: MESA_DISK_CACHE_DATABASE_EVICTION_SCORE_2X_PERIOD
+
+   Mesa-DB cache eviction algorithm calculates weighted score for the
+   cache items. The weight is doubled based on the last access time of
+   cache entry. By default period of weight doubling is set to one month.
+   Period value is given in seconds.
+
 .. envvar:: MESA_DISK_CACHE_READ_ONLY_FOZ_DBS_DYNAMIC_LIST
 
    if set with :envvar:`MESA_DISK_CACHE_SINGLE_FILE` enabled, references

src/util/mesa_cache_db.c

@@ -25,6 +25,7 @@
 #include "mesa_cache_db.h"
 #include "os_time.h"
 #include "ralloc.h"
+#include "u_debug.h"
 #include "u_qsort.h"
 
 #define MESA_CACHE_DB_VERSION          1
@@ -398,6 +399,19 @@ mesa_db_close_file(struct mesa_cache_db_file *db_file)
    free(db_file->path);
 }
 
+static bool
+mesa_db_remove_file(struct mesa_cache_db_file *db_file,
+                  const char *cache_path,
+                  const char *filename)
+{
+   if (asprintf(&db_file->path, "%s/%s", cache_path, filename) == -1)
+      return false;
+
+   unlink(db_file->path);
+
+   return true;
+}
+
 static int
 entry_sort_lru(const void *_a, const void *_b, void *arg)
 {
@@ -625,6 +639,22 @@ close_cache:
    return false;
 }
 
+bool
+mesa_db_wipe_path(const char *cache_path)
+{
+   struct mesa_cache_db db = {0};
+   bool success = true;
+
+   if (!mesa_db_remove_file(&db.cache, cache_path, "mesa_cache.db") ||
+       !mesa_db_remove_file(&db.index, cache_path, "mesa_cache.idx"))
+      success = false;
+
+   free(db.cache.path);
+   free(db.index.path);
+
+   return success;
+}
+
 void
 mesa_cache_db_close(struct mesa_cache_db *db)
 {
@@ -870,4 +900,100 @@ fail:
    return false;
 }
 
+bool
+mesa_cache_db_has_space(struct mesa_cache_db *db, size_t blob_size)
+{
+   bool has_space;
+
+   if (!mesa_db_lock(db))
+      return false;
+
+   if (!mesa_db_seek_end(db->cache.file))
+      goto fail_fatal;
+
+   has_space = mesa_cache_db_has_space_locked(db, blob_size);
+
+   mesa_db_unlock(db);
+
+   return has_space;
+
+fail_fatal:
+   mesa_db_zap(db);
+   mesa_db_unlock(db);
+
+   return false;
+}
+
+static uint64_t
+mesa_cache_db_eviction_2x_score_period(void)
+{
+   const uint64_t nsec_per_sec = 1000000000ull;
+   static uint64_t period = 0;
+
+   if (period)
+      return period;
+
+   period = debug_get_num_option("MESA_DISK_CACHE_DATABASE_EVICTION_SCORE_2X_PERIOD",
+                                 30 * 24 * 60 * 60) * nsec_per_sec;
+
+   return period;
+}
+
+double
+mesa_cache_db_eviction_score(struct mesa_cache_db *db)
+{
+   int64_t eviction_size = mesa_cache_db_eviction_size(db);
+   struct mesa_index_db_hash_entry **entries;
+   unsigned num_entries, i = 0;
+   double eviction_score = 0;
+
+   if (!mesa_db_lock(db))
+      return 0;
+
+   if (!db->alive)
+      goto fail;
+
+   if (!mesa_db_reload(db))
+      goto fail_fatal;
+
+   num_entries = _mesa_hash_table_num_entries(db->index_db->table);
+   entries = calloc(num_entries, sizeof(*entries));
+   if (!entries)
+      goto fail;
+
+   hash_table_foreach(db->index_db->table, entry)
+      entries[i++] = entry->data;
+
+   util_qsort_r(entries, num_entries, sizeof(*entries),
+                entry_sort_lru, db);
+
+   for (i = 0; eviction_size > 0 && i < num_entries; i++) {
+      uint64_t entry_age = os_time_get_nano() - entries[i]->last_access_time;
+      unsigned entry_size = blob_file_size(entries[i]->size);
+
+      /* Eviction score is a sum of weighted cache entry sizes,
+       * where weight doubles for each month of entry's age.
+       */
+      uint64_t period = mesa_cache_db_eviction_2x_score_period();
+      double entry_scale = 1 + (double)entry_age / period;
+      double entry_score = entry_size * entry_scale;
+
+      eviction_score += entry_score;
+      eviction_size -= entry_size;
+   }
+
+   free(entries);
+
+   mesa_db_unlock(db);
+
+   return eviction_score;
+
+fail_fatal:
+   mesa_db_zap(db);
+fail:
+   mesa_db_unlock(db);
+
+   return 0;
+}
+
 #endif /* DETECT_OS_WINDOWS */

src/util/mesa_cache_db.h

@@ -66,6 +66,15 @@ mesa_cache_db_entry_write(struct mesa_cache_db *db,
 bool
 mesa_cache_db_entry_remove(struct mesa_cache_db *db,
                            const uint8_t *cache_key_160bit);
+
+bool
+mesa_db_wipe_path(const char *cache_path);
+
+bool
+mesa_cache_db_has_space(struct mesa_cache_db *db, size_t blob_size);
+
+double
+mesa_cache_db_eviction_score(struct mesa_cache_db *db);
 #else
 static inline bool
 mesa_cache_db_open(struct mesa_cache_db *db, const char *cache_path)
@@ -112,6 +121,24 @@ mesa_cache_db_entry_remove(struct mesa_cache_db *db,
 {
    return false;
 }
+
+static inline bool
+mesa_db_wipe_path(const char *cache_path)
+{
+   return false;
+}
+
+static inline bool
+mesa_cache_db_has_space(struct mesa_cache_db *db, size_t blob_size)
+{
+   return false;
+}
+
+static inline double
+mesa_cache_db_eviction_score(struct mesa_cache_db *db)
+{
+   return 0;
+}
 #endif /* DETECT_OS_WINDOWS */
 
 #ifdef __cplusplus

src/util/mesa_cache_db_multipart.c

@@ -0,0 +1,160 @@
+/*
+ * Copyright © 2022 Collabora, Ltd.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <sys/stat.h>
+
+#include "detect_os.h"
+#include "string.h"
+#include "mesa_cache_db_multipart.h"
+#include "u_debug.h"
+
+bool
+mesa_cache_db_multipart_open(struct mesa_cache_db_multipart *db,
+                             const char *cache_path)
+{
+#if DETECT_OS_WINDOWS
+   return false;
+#else
+   char *part_path = NULL;
+   unsigned int i;
+
+   db->num_parts = debug_get_num_option("MESA_DISK_CACHE_DATABASE_NUM_PARTS", 50);
+
+   db->parts = calloc(db->num_parts, sizeof(*db->parts));
+   if (!db->parts)
+      return false;
+
+   for (i = 0; i < db->num_parts; i++) {
+      bool db_opened = false;
+
+      if (asprintf(&part_path, "%s/part%u", cache_path, i) == -1)
+         goto close_db;
+
+      if (mkdir(part_path, 0755) == -1 && errno != EEXIST)
+         goto free_path;
+
+      /* DB opening may fail only in a case of a severe problem,
+       * like IO error.
+       */
+      db_opened = mesa_cache_db_open(&db->parts[i], part_path);
+      if (!db_opened)
+         goto free_path;
+
+      free(part_path);
+   }
+
+   /* remove old pre multi-part cache */
+   mesa_db_wipe_path(cache_path);
+
+   return true;
+
+free_path:
+   free(part_path);
+close_db:
+   while (i--)
+      mesa_cache_db_close(&db->parts[i]);
+
+   free(db->parts);
+
+   return false;
+#endif
+}
+
+void
+mesa_cache_db_multipart_close(struct mesa_cache_db_multipart *db)
+{
+   while (db->num_parts--)
+      mesa_cache_db_close(&db->parts[db->num_parts]);
+
+   free(db->parts);
+}
+
+void
+mesa_cache_db_multipart_set_size_limit(struct mesa_cache_db_multipart *db,
+                                       uint64_t max_cache_size)
+{
+   for (unsigned int i = 0; i < db->num_parts; i++)
+      mesa_cache_db_set_size_limit(&db->parts[i],
+                                   max_cache_size / db->num_parts);
+}
+
+void *
+mesa_cache_db_multipart_read_entry(struct mesa_cache_db_multipart *db,
+                                   const uint8_t *cache_key_160bit,
+                                   size_t *size)
+{
+   unsigned last_read_part = db->last_read_part;
+
+   for (unsigned int i = 0; i < db->num_parts; i++) {
+      unsigned int part = (last_read_part + i) % db->num_parts;
+
+      void *cache_item = mesa_cache_db_read_entry(&db->parts[part],
+                                                  cache_key_160bit, size);
+      if (cache_item) {
+         /* Likely that the next entry lookup will hit the same DB part. */
+         db->last_read_part = part;
+         return cache_item;
+      }
+   }
+
+   return NULL;
+}
+
+static unsigned
+mesa_cache_db_multipart_select_victim_part(struct mesa_cache_db_multipart *db)
+{
+   double best_score = 0, score;
+   unsigned victim = 0;
+
+   for (unsigned int i = 0; i < db->num_parts; i++) {
+      score = mesa_cache_db_eviction_score(&db->parts[i]);
+      if (score > best_score) {
+         best_score = score;
+         victim = i;
+      }
+   }
+
+   return victim;
+}
+
+bool
+mesa_cache_db_multipart_entry_write(struct mesa_cache_db_multipart *db,
+                                    const uint8_t *cache_key_160bit,
+                                    const void *blob, size_t blob_size)
+{
+   unsigned last_written_part = db->last_written_part;
+   int wpart = -1;
+
+   for (unsigned int i = 0; i < db->num_parts; i++) {
+      unsigned int part = (last_written_part + i) % db->num_parts;
+
+      /* Note that each DB part has own locking. */
+      if (mesa_cache_db_has_space(&db->parts[part], blob_size)) {
+         wpart = part;
+         break;
+      }
+   }
+
+   /* All DB parts are full. Writing to a full DB part will auto-trigger
+    * eviction of LRU cache entries from the part. Select DB part that
+    * contains majority of LRU cache entries.
+    */
+   if (wpart < 0)
+      wpart = mesa_cache_db_multipart_select_victim_part(db);
+
+   db->last_written_part = wpart;
+
+   return mesa_cache_db_entry_write(&db->parts[wpart], cache_key_160bit,
+                                    blob, blob_size);
+}
+
+void
+mesa_cache_db_multipart_entry_remove(struct mesa_cache_db_multipart *db,
+                                     const uint8_t *cache_key_160bit)
+{
+   for (unsigned int i = 0; i < db->num_parts; i++)
+      mesa_cache_db_entry_remove(&db->parts[i], cache_key_160bit);
+}

src/util/mesa_cache_db_multipart.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright © 2022 Collabora, Ltd.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef MESA_CACHE_DB_MULTIPART_H
+#define MESA_CACHE_DB_MULTIPART_H
+
+#include "mesa_cache_db.h"
+
+struct mesa_cache_db_multipart {
+   struct mesa_cache_db *parts;
+   unsigned int num_parts;
+   volatile unsigned int last_read_part;
+   volatile unsigned int last_written_part;
+};
+
+bool
+mesa_cache_db_multipart_open(struct mesa_cache_db_multipart *db,
+                             const char *cache_path);
+
+void
+mesa_cache_db_multipart_close(struct mesa_cache_db_multipart *db);
+
+void
+mesa_cache_db_multipart_set_size_limit(struct mesa_cache_db_multipart *db,
+                                       uint64_t max_cache_size);
+
+void *
+mesa_cache_db_multipart_read_entry(struct mesa_cache_db_multipart *db,
+                                   const uint8_t *cache_key_160bit,
+                                   size_t *size);
+
+bool
+mesa_cache_db_multipart_entry_write(struct mesa_cache_db_multipart *db,
+                                    const uint8_t *cache_key_160bit,
+                                    const void *blob, size_t blob_size);
+
+void
+mesa_cache_db_multipart_entry_remove(struct mesa_cache_db_multipart *db,
+                                     const uint8_t *cache_key_160bit);
+
+#endif /* MESA_CACHE_DB_MULTIPART_H */

src/util/meson.build

@@ -164,6 +164,8 @@ files_mesa_util = files(
   'xxhash.h',
   'mesa_cache_db.c',
   'mesa_cache_db.h',
+  'mesa_cache_db_multipart.c',
+  'mesa_cache_db_multipart.h',
 )
 
 files_drirc = files('00-mesa-defaults.conf')

src/util/tests/cache_test.cpp

@@ -762,6 +762,7 @@ TEST_F(Cache, Database)
 #ifndef ENABLE_SHADER_CACHE
    GTEST_SKIP() << "ENABLE_SHADER_CACHE not defined.";
 #else
+   setenv("MESA_DISK_CACHE_DATABASE_NUM_PARTS", "1", 1);
    setenv("MESA_DISK_CACHE_DATABASE", "true", 1);
 
    test_disk_cache_create(mem_ctx, CACHE_DIR_NAME_DB, driver_id);
@@ -787,6 +788,7 @@ TEST_F(Cache, Database)
    test_put_and_get_between_instances_with_eviction(driver_id);
 
    setenv("MESA_DISK_CACHE_DATABASE", "false", 1);
+   unsetenv("MESA_DISK_CACHE_DATABASE_NUM_PARTS");
 
    err = rmrf_local(CACHE_TEST_TMP);
    EXPECT_EQ(err, 0) << "Removing " CACHE_TEST_TMP " again";
@@ -1172,3 +1174,105 @@ TEST_F(Cache, List)
 #endif /* FOZ_DB_UTIL_DYNAMIC_LIST */
 #endif /* ENABLE_SHADER_CACHE */
 }
+
+static void
+test_multipart_eviction(const char *driver_id)
+{
+   const unsigned int entry_size = 512;
+   uint8_t blobs[7][entry_size];
+   cache_key keys[7];
+   unsigned int i;
+   char *result;
+   size_t size;
+
+   setenv("MESA_SHADER_CACHE_MAX_SIZE", "3K", 1);
+   setenv("MESA_DISK_CACHE_DATABASE_EVICTION_SCORE_2X_PERIOD", "1", 1);
+
+   struct disk_cache *cache = disk_cache_create("test", driver_id, 0);
+
+   unsigned int entry_file_size = entry_size;
+   entry_file_size -= sizeof(struct cache_entry_file_data);
+   entry_file_size -= mesa_cache_db_file_entry_size();
+   entry_file_size -= cache->driver_keys_blob_size;
+   entry_file_size -= 4 + 8; /* cache_item_metadata size + room for alignment */
+
+   /*
+    * 1. Allocate 3KB cache in 3 parts, each part is 1KB
+    * 2. Fill up cache with six 512K entries
+    * 3. Touch entries of the first part, which will bump last_access_time
+    *    of the first two cache entries
+    * 4. Insert seventh 512K entry that will cause eviction of the second part
+    * 5. Check that second entry of the second part gone due to eviction and
+    *    others present
+    */
+
+   /* Fill up cache with six 512K entries. */
+   for (i = 0; i < 6; i++) {
+      memset(blobs[i], i, entry_file_size);
+
+      disk_cache_compute_key(cache,  blobs[i], entry_file_size, keys[i]);
+      disk_cache_put(cache, keys[i], blobs[i], entry_file_size, NULL);
+      disk_cache_wait_for_idle(cache);
+
+      result = (char *) disk_cache_get(cache, keys[i], &size);
+      EXPECT_NE(result, nullptr) << "disk_cache_get with existent item (pointer)";
+      EXPECT_EQ(size, entry_file_size) << "disk_cache_get with existent item (size)";
+      free(result);
+
+      /* Ensure that cache entries will have distinct last_access_time
+       * during testing.
+       */
+      if (i % 2 == 0)
+         usleep(100000);
+   }
+
+   /* Touch entries of the first part. Second part becomes outdated */
+   for (i = 0; i < 2; i++) {
+      result = (char *) disk_cache_get(cache, keys[i], &size);
+      EXPECT_NE(result, nullptr) << "disk_cache_get with existent item (pointer)";
+      EXPECT_EQ(size, entry_file_size) << "disk_cache_get with existent item (size)";
+      free(result);
+   }
+
+   /* Insert seventh entry. */
+   memset(blobs[6], 6, entry_file_size);
+   disk_cache_compute_key(cache,  blobs[6], entry_file_size, keys[6]);
+   disk_cache_put(cache, keys[6], blobs[6], entry_file_size, NULL);
+   disk_cache_wait_for_idle(cache);
+
+   /* Check whether third entry of the second part gone and others present. */
+   for (i = 0; i < ARRAY_SIZE(blobs); i++) {
+      result = (char *) disk_cache_get(cache, keys[i], &size);
+      if (i == 2) {
+         EXPECT_EQ(result, nullptr) << "disk_cache_get with non-existent item (pointer)";
+      } else {
+         EXPECT_NE(result, nullptr) << "disk_cache_get with existent item (pointer)";
+         EXPECT_EQ(size, entry_file_size) << "disk_cache_get with existent item (size)";
+      }
+      free(result);
+   }
+
+   disk_cache_destroy(cache);
+}
+
+TEST_F(Cache, DatabaseMultipartEviction)
+{
+   const char *driver_id = "make_check_uncompressed";
+
+#ifndef ENABLE_SHADER_CACHE
+   GTEST_SKIP() << "ENABLE_SHADER_CACHE not defined.";
+#else
+   setenv("MESA_DISK_CACHE_DATABASE_NUM_PARTS", "3", 1);
+   setenv("MESA_DISK_CACHE_DATABASE", "true", 1);
+
+   test_disk_cache_create(mem_ctx, CACHE_DIR_NAME_DB, driver_id);
+
+   test_multipart_eviction(driver_id);
+
+   unsetenv("MESA_DISK_CACHE_DATABASE_NUM_PARTS");
+   unsetenv("MESA_DISK_CACHE_DATABASE");
+
+   int err = rmrf_local(CACHE_TEST_TMP);
+   EXPECT_EQ(err, 0) << "Removing " CACHE_TEST_TMP " again";
+#endif
+}