platform/tests: add test for cache consistency for IP routes

Routes can be added with `ip route add|change|replace|append|prepend`.
Add a test that randomly tries to add such routes, and checks that
the cache stays consistent.

https://bugzilla.redhat.com/show_bug.cgi?id=2060684

2 files changed, 230 insertions, 1 deletions

diff --git a/src/core/platform/tests/test-common.c b/src/core/platform/tests/test-common.c
index 0ca84cd999..e9c36a52dd 100644
--- a/src/core/platform/tests/test-common.c
+++ b/src/core/platform/tests/test-common.c
@@ -1028,7 +1028,16 @@ nmtstp_assert_platform(NMPlatform *platform, guint32 obj_type_flags)
                      * doesn't work. We now always request a new dump. */
                 } else if (obj_type == NMP_OBJECT_TYPE_IP4_ROUTE) {
                     /* For IPv4, it also does not reliably always work. This may
-                     * be a bug we want to fix. For now, ignore the check. */
+                     * be a bug we want to fix. For now, ignore the check.
+                     *
+                     * This is probably caused by kernel bug
+                     * https://bugzilla.redhat.com/show_bug.cgi?id=2162315
+                     * for which I think there is no workaround.
+                     *
+                     * Also, rhbz#2162315 means NMPlatform will merge two different
+                     * routes together, if one of them were deleted, the RTM_DELROUTE
+                     * message would wrongly delete single entry, leading to cache
+                     * inconsistency. */
                 } else {
                     /* Assert that also the original, not-sorted lists agree. */
                     _assert_platform_compare_arr(obj_type,
diff --git a/src/core/platform/tests/test-route.c b/src/core/platform/tests/test-route.c
index 9939ccdc95..001ecbc70f 100644
--- a/src/core/platform/tests/test-route.c
+++ b/src/core/platform/tests/test-route.c
@@ -2160,6 +2160,213 @@ test_mptcp(gconstpointer test_data)
 
 /*****************************************************************************/
 
+static void
+_ensure_onlink_routes(void)
+{
+    int i;
+
+    for (i = 0; i < G_N_ELEMENTS(NMTSTP_ENV1_DEVICE_NAME) && NMTSTP_ENV1_DEVICE_NAME[i]; i++) {
+        nmtstp_run_command("ip route append 7.7.7.0/24 dev %s", NMTSTP_ENV1_DEVICE_NAME[i]);
+        nmtstp_run_command("ip route append 7:7:7::/64 dev %s", NMTSTP_ENV1_DEVICE_NAME[i]);
+    }
+}
+
+static void
+test_cache_consistency_routes(gconstpointer test_data)
+{
+    const int                    TEST_IDX      = GPOINTER_TO_INT(test_data);
+    NMPlatform                  *platform      = NM_PLATFORM_GET;
+    gboolean                     is_test_quick = nmtst_test_quick();
+    const int                    N_RUN         = is_test_quick ? 50 : 500;
+    int                          i_run;
+    gs_unref_ptrarray GPtrArray *keeper = g_ptr_array_new_with_free_func(g_free);
+
+    _ensure_onlink_routes();
+
+    for (i_run = 0; i_run < N_RUN; i_run++) {
+        const char   *extra_options[100];
+        gsize         n_extra_options   = 0;
+        gs_free char *extra_options_str = NULL;
+        int           i_if;
+        int           ifindex;
+        const char   *ifname;
+        int           IS_IPv4;
+        const char   *op;
+        const char   *prefix;
+        const char   *s;
+        const char   *route_type;
+        int           i;
+        int           n;
+        char          addr_family_char[2] = {'6', '4'};
+
+        g_ptr_array_set_size(keeper, 0);
+
+        switch (TEST_IDX) {
+        case 1:
+            IS_IPv4 = TRUE;
+            break;
+        case 2:
+            IS_IPv4 = FALSE;
+            break;
+        default:
+            IS_IPv4 = nmtst_get_rand_bool();
+            break;
+        }
+
+        i_if = nmtst_get_rand_uint32() % 2;
+        op   = nmtst_rand_select_str("flush", "add", "change", "append", "prepend", "replace");
+
+        ifindex = NMTSTP_ENV1_IFINDEXES[i_if];
+        ifname  = NMTSTP_ENV1_DEVICE_NAME[i_if];
+
+        g_assert_cmpint(ifindex, ==, nm_platform_link_get_ifindex(platform, ifname));
+
+        if (nm_streq(op, "flush")) {
+            if (!nmtst_get_rand_one_case_in(10)) {
+                /* flush more seldom. */
+                continue;
+            }
+            nmtstp_run_command("ip -%c route flush dev %s"
+                               "%s" /* redirect */
+                               "",
+                               addr_family_char[IS_IPv4],
+                               ifname,
+                               nmtst_is_debug() ? "" : " &>/dev/null");
+            _ensure_onlink_routes();
+            goto done;
+        }
+
+        route_type = nmtst_get_rand_one_case_in(4)
+                         ? nmtst_rand_select_str("unicast", "blackhole", "local", "broadcast")
+                         : NULL;
+
+        if (NM_IN_STRSET(route_type, "blackhole")) {
+            ifindex = 0;
+            ifname  = NULL;
+        }
+
+        if (IS_IPv4) {
+            prefix = nmtst_rand_select_str("192.168.4.0/24",
+                                           "192.168.5.0/24",
+                                           "192.168.5.5/32",
+                                           "default");
+        } else {
+            prefix =
+                nmtst_rand_select_str("a:b:c:d::/64", "a:b:c:e::/64", "a:b:c:f::/64", "default");
+        }
+
+        s = nmtst_rand_select_str(NULL, "kernel", "bird");
+        if (s) {
+            if (nmtst_get_rand_bool()) {
+                s = nm_streq(s, "kernel") ? nmtst_rand_select_str("boot", "static", "ra")
+                                          : nmtst_rand_select_str("babel", "bgp");
+            }
+            extra_options[n_extra_options++] = "proto";
+            extra_options[n_extra_options++] = s;
+        }
+
+        s = nmtst_rand_select_str(NULL, "10", "20");
+        if (s) {
+            extra_options[n_extra_options++] = "metric";
+            extra_options[n_extra_options++] = s;
+        }
+
+        s = nmtst_rand_select_str(NULL, "10222", "10223");
+        if (s) {
+            extra_options[n_extra_options++] = "table";
+            extra_options[n_extra_options++] = s;
+        }
+
+        if (!IS_IPv4 && NM_IN_STRSET(op, "add", "change", "append", "prepend", "replace")) {
+            /* kernel has a bug with append/prepend of IPv6 routes with next-hops.
+             * This leads to wrong notification messages, wrong merging of multi-hop
+             * routes and cache inconsistency in NMPlatform.
+             *
+             * https://bugzilla.redhat.com/show_bug.cgi?id=2161994
+             *
+             * For now, disable the test case to make the unit test not fail.
+             *
+             * While being a kernel bug, it leads to cache inconsistency in NMPlatform,
+             * which is a problem for NetworkManager. I don't see how we can detect
+             * this problem to trigger a refresh. */
+        } else if (ifname && nmtst_get_rand_one_case_in(3)) {
+            n = (nmtst_get_rand_uint32() % 4) + 1;
+            for (i = 0; i < n; i++) {
+                extra_options[n_extra_options++] = "nexthop";
+                extra_options[n_extra_options++] = "via";
+                if (IS_IPv4) {
+                    extra_options[n_extra_options++] =
+                        nmtst_keeper_printf(&keeper, "7.7.7.%d", i + 1);
+                } else {
+                    extra_options[n_extra_options++] =
+                        nmtst_keeper_printf(&keeper, "7:7:7:7::%d", i + 1);
+                }
+                extra_options[n_extra_options++] = "dev";
+                extra_options[n_extra_options++] = NMTSTP_ENV1_DEVICE_NAME[nmtst_get_rand_bool()];
+                if (nmtst_get_rand_one_case_in(3)) {
+                    extra_options[n_extra_options++] = "weight";
+                    extra_options[n_extra_options++] = "5";
+                }
+            }
+
+            ifname  = NULL;
+            ifindex = 0;
+        }
+
+        g_assert_cmpint(n_extra_options, <, G_N_ELEMENTS(extra_options));
+        extra_options[n_extra_options] = NULL;
+
+        if (nmtst_is_debug())
+            nmtstp_run_command("ip -%c -d route show table all", addr_family_char[IS_IPv4]);
+
+        /* We ignore errors. The reason is that operations like "change" might fail if
+         * the route doesn't exist. That's fine for our test. We just do randomly things
+         * and some of them will stick. */
+        nmtstp_run_command(
+            "ip -%c route "
+            "%s"   /* op */
+            "%s%s" /* route_type */
+            " %s"  /* prefix */
+            "%s%s" /* ifname */
+            "%s%s" /* extra_options */
+            "%s"   /* redirect */
+            "",
+            addr_family_char[IS_IPv4],
+            op,
+            NM_PRINT_FMT_QUOTED2(route_type, " ", route_type, ""),
+            prefix,
+            NM_PRINT_FMT_QUOTED2(ifname, " dev ", ifname, ""),
+            NM_PRINT_FMT_QUOTED2(extra_options[0],
+                                 " ",
+                                 (extra_options_str = g_strjoinv(" ", (char **) extra_options)),
+                                 ""),
+            nmtst_is_debug() ? "" : " &>/dev/null");
+
+        if (nmtst_is_debug())
+            nmtstp_run_command("ip -%c -d route show table all", addr_family_char[IS_IPv4]);
+done:
+        nm_platform_process_events(platform);
+
+        if (!is_test_quick || (i_run + 1 == N_RUN) || nmtst_get_rand_one_case_in(5)) {
+            nmtstp_assert_platform(
+                platform,
+                nmtst_get_rand_one_case_in(5)
+                    ? 0u
+                    : nmp_object_type_to_flags(NMP_OBJECT_TYPE_IP_ROUTE(IS_IPv4)));
+        }
+    }
+
+    if (is_test_quick) {
+        gs_free char *msg = NULL;
+
+        msg = g_strdup_printf("Ran a quick version of test %s (try NMTST_DEBUG=slow)",
+                              nmtst_test_get_path());
+        g_test_skip(msg);
+    }
+}
+
+/*****************************************************************************/
+
 NMTstpSetupFunc const _nmtstp_setup_platform_func = SETUP;
 
 void
@@ -2174,6 +2381,8 @@ _nmtstp_setup_tests(void)
 #define add_test_func(testpath, test_func) nmtstp_env1_add_test_func(testpath, test_func, 1, TRUE)
 #define add_test_func_data(testpath, test_func, arg) \
     nmtstp_env1_add_test_func_data(testpath, test_func, arg, 1, TRUE)
+#define add_test_func_data_with_if2(testpath, test_func, arg) \
+    nmtstp_env1_add_test_func_data(testpath, test_func, arg, 2, TRUE)
 
     add_test_func("/route/ip4", test_ip4_route);
     add_test_func("/route/ip6", test_ip6_route);
@@ -2206,4 +2415,15 @@ _nmtstp_setup_tests(void)
         add_test_func_data("/route/mptcp/1", test_mptcp, GINT_TO_POINTER(1));
         add_test_func_data("/route/mptcp/2", test_mptcp, GINT_TO_POINTER(2));
     }
+    if (nmtstp_is_root_test()) {
+        add_test_func_data_with_if2("/route/test_cache_consistency_routes/1",
+                                    test_cache_consistency_routes,
+                                    GINT_TO_POINTER(1));
+        add_test_func_data_with_if2("/route/test_cache_consistency_routes/2",
+                                    test_cache_consistency_routes,
+                                    GINT_TO_POINTER(2));
+        add_test_func_data_with_if2("/route/test_cache_consistency_routes/3",
+                                    test_cache_consistency_routes,
+                                    GINT_TO_POINTER(3));
+    }
 }