MEDIUM: proxy: implement persistent named defaults

This patch changes the handling of named defaults sections. Prior to
this patch, every unreferenced defaults proxies were removed on post
parsing. Now by default, these sections are kept after postparsing and
only purged on deinit. The objective is to allow reusing them as base
configuration for dynamic backends.

To implement this, refcount of every still addressable named sections is
incremented by one after parsing. This ensures that they won't be
removed even if referencing proxies are removed at runtime. This is done
via the new function proxy_ref_all_defaults().

To ensure defaults instances are still properly removed on deinit, the
inverse operation is performed : refcount is decremented by one on every
defaults sections via proxy_unref_all_defaults().

The original behavior can still be used by using the new global keyword
tune.defaults.purge. This is useful for users using configuration with
large number of defaults and not interested in dynamic backends
creation.

doc/configuration.txt

@@ -1871,6 +1871,7 @@ The following keywords are supported in the "global" section :
    - tune.bufsize
    - tune.bufsize.small
    - tune.comp.maxlevel
+   - tune.defaults.purge
    - tune.disable-fast-forward
    - tune.disable-zero-copy-forwarding
    - tune.epoll.mask-events
@@ -4121,6 +4122,19 @@ tune.comp.maxlevel <number>
   Each stream using compression initializes the compression algorithm with
   this value. The default value is 1.
 
+tune.defaults.purge
+  For dynamic backends support, all named defaults sections are now kept in
+  memory after parsing. This is necessary as backend added at runtime must be
+  based on a named defaults for its configuration.
+
+  This may consume significant memory if the number of defaults instances is
+  important. In this case and if dynamic backend feature is unnecessary, it's
+  possible to use this option to force deletion of defaults section after
+  parsing. It is still mandatory though to keep referenced defaults section
+  which contain settings whose cannot be copied by their referencing proxies.
+  For example, this is the case if the defaults section defines TCP/HTTP rules
+  or a tcpcheck ruleset.
+
 tune.disable-fast-forward
   Disables the data fast-forwarding. It is a mechanism to optimize the data
   forwarding by passing data directly from a side to the other one without

include/haproxy/global-t.h

@@ -67,7 +67,7 @@
 #define GTUNE_USE_SYSTEMD        (1<<10)
 
 #define GTUNE_BUSY_POLLING       (1<<11)
-/* (1<<12) unused */
+#define GTUNE_PURGE_DEFAULTS     (1<<12)
 #define GTUNE_SET_DUMPABLE       (1<<13)
 #define GTUNE_USE_EVPORTS        (1<<14)
 #define GTUNE_STRICT_LIMITS      (1<<15)

include/haproxy/proxy.h

@@ -72,6 +72,8 @@ void init_new_proxy(struct proxy *p);
 void defaults_px_destroy(struct proxy *px);
 void defaults_px_destroy_all_unref(void);
 void defaults_px_detach(struct proxy *px);
+void defaults_px_ref_all(void);
+void defaults_px_unref_all(void);
 
 void proxy_ref_defaults(struct proxy *px, struct proxy *defpx);
 void proxy_unref_defaults(struct proxy *px);

src/cfgparse-global.c

@@ -1423,6 +1423,9 @@ static int cfg_parse_global_tune_opts(char **args, int section_type,
 
 		return 0;
 	}
+	else if (strcmp(args[0], "tune.defaults.purge") == 0) {
+		global.tune.options |= GTUNE_PURGE_DEFAULTS;
+	}
 	else if (strcmp(args[0], "tune.pattern.cache-size") == 0) {
 		if (*(args[1]) == 0) {
 			memprintf(err, "'%s' expects a positive numeric value", args[0]);
@@ -1869,6 +1872,7 @@ static struct cfg_kw_list cfg_kws = {ILH, {
 	{ CFG_GLOBAL, "tune.bufsize", cfg_parse_global_tune_opts },
 	{ CFG_GLOBAL, "tune.chksize", cfg_parse_global_unsupported_opts },
 	{ CFG_GLOBAL, "tune.comp.maxlevel", cfg_parse_global_tune_opts },
+	{ CFG_GLOBAL, "tune.defaults.purge", cfg_parse_global_tune_opts },
 	{ CFG_GLOBAL, "tune.disable-fast-forward", cfg_parse_global_tune_forward_opts },
 	{ CFG_GLOBAL, "tune.disable-zero-copy-forwarding", cfg_parse_global_tune_forward_opts },
 	{ CFG_GLOBAL, "tune.glitches.kill.cpu-usage", cfg_parse_global_tune_opts },

src/haproxy.c

@@ -2105,8 +2105,13 @@ static void step_init_2(int argc, char** argv)
 	}
 	last_defproxy = NULL; /* This variable is not used after parsing. */
 
-	/* destroy unreferenced defaults proxies  */
-	defaults_px_destroy_all_unref();
+	if (global.tune.options & GTUNE_PURGE_DEFAULTS) {
+		/* destroy unreferenced defaults proxies  */
+		defaults_px_destroy_all_unref();
+	}
+	else {
+		defaults_px_ref_all();
+	}
 
 	list_for_each_entry(prcf, &pre_check_list, list) {
 		err_code |= prcf->fct();
@@ -2749,6 +2754,9 @@ void deinit(void)
 	 * they are respectively cleaned up in sink_deinit() and deinit_log_forward()
 	 */
 
+	/* If named defaults were preserved, ensure refcount is resetted. */
+	if (!(global.tune.options & GTUNE_PURGE_DEFAULTS))
+		defaults_px_unref_all();
 	/* All proxies are removed now, so every defaults should also be freed
 	 * when their refcount reached zero.
 	 */

src/proxy.c

@@ -1667,6 +1667,30 @@ void defaults_px_detach(struct proxy *px)
 	/* If not destroyed, <px> can still be accessed in <defaults_list>. */
 }
 
+void defaults_px_ref_all(void)
+{
+	struct proxy *px;
+
+	for (px = cebis_item_first(&defproxy_by_name, conf.name_node, id, struct proxy);
+	     px;
+	     px = cebis_item_next(&defproxy_by_name, conf.name_node, id, px)) {
+		++px->conf.refcount;
+	}
+}
+
+void defaults_px_unref_all(void)
+{
+	struct proxy *px, *nx;
+
+	for (px = cebis_item_first(&defproxy_by_name, conf.name_node, id, struct proxy); px; px = nx) {
+		nx = cebis_item_next(&defproxy_by_name, conf.name_node, id, px);
+
+		BUG_ON(!px->conf.refcount);
+		if (!--px->conf.refcount)
+			defaults_px_destroy(px);
+	}
+}
+
 /* Add a reference on the default proxy <defpx> for the proxy <px> Nothing is
  * done if <px> already references <defpx>. Otherwise, the default proxy
  * refcount is incremented by one. For now, this operation is not thread safe