MINOR: dynbuf: pass a criticality argument to b_alloc()

The goal is to indicate how critical the allocation is, between the
least one (growing an existing buffer ring) and the topmost one (boot
time allocation for the life of the process).

The 3 tcp-based muxes (h1, h2, fcgi) use a common allocation function
to try to allocate otherwise subscribe. There's currently no distinction
of direction nor part that tries to allocate, and this should be revisited
to improve this situation, particularly when we consider that mux-h2 can
reduce its Tx allocations if needed.

For now, 4 main levels are planned, to translate how the data travels
inside haproxy from a producer to a consumer:
  - MUX_RX:   buffer used to receive data from the OS
  - SE_RX:    buffer used to place a transformation of the RX data for
              a mux, or to produce a response for an applet
  - CHANNEL:  the channel buffer for sync recv
  - MUX_TX:   buffer used to transfer data from the channel to the outside,
              generally a mux but there can be a few specificities (e.g.
              http client's response buffer passed to the application,
              which also gets a transformation of the channel data).

The other levels are a bit different in that they don't strictly need to
allocate for the first two ones, or they're permanent for the last one
(used by compression).

include/haproxy/applet.h

@@ -91,7 +91,7 @@ static inline struct buffer *appctx_get_buf(struct appctx *appctx, struct buffer
 	struct buffer *buf = NULL;
 
 	if (likely(!LIST_INLIST(&appctx->buffer_wait.list)) &&
-	    unlikely((buf = b_alloc(bptr)) == NULL)) {
+	    unlikely((buf = b_alloc(bptr, DB_CHANNEL)) == NULL)) {
 		appctx->buffer_wait.target = appctx;
 		appctx->buffer_wait.wakeup_cb = appctx_buf_available;
 		LIST_APPEND(&th_ctx->buffer_wq, &appctx->buffer_wait.list);

include/haproxy/channel.h

@@ -915,7 +915,7 @@ static inline int ci_space_for_replace(const struct channel *chn)
  */
 static inline int channel_alloc_buffer(struct channel *chn, struct buffer_wait *wait)
 {
-	if (b_alloc(&chn->buf) != NULL)
+	if (b_alloc(&chn->buf, DB_CHANNEL) != NULL)
 		return 1;
 
 	if (!LIST_INLIST(&wait->list))

include/haproxy/dynbuf-t.h

@@ -24,6 +24,41 @@
 
 #include <haproxy/list-t.h>
 
+/* Describe the levels of criticality of each allocation based on the expected
+ * use case. We distinguish multiple use cases, from the least important to the
+ * most important one:
+ *   - allocate a buffer to grow a non-empty ring: this should be avoided when
+ *     resources are becoming scarce.
+ *   - allocate a buffer for very unlikely situations (e.g. L7 retries, early
+ *     data). These may acceptably fail on low resources.
+ *   - buffer used to receive data in the mux at the connection level. Please
+ *     note that this level might later be resplit into two levels, one for
+ *     initial data such as a new request, which may be rejected and postponed,
+ *     and one for data continuation, which may be needed to complete a request
+ *     or receive some control data allowing another buffer to be flushed.
+ *   - buffer used to produce data at the endpoint for internal consumption,
+ *     typically mux streams and applets. These buffers will be allocated until
+ *     a channel picks them. Not processing them might sometimes lead to a mux
+ *     being clogged and blocking other streams from progressing.
+ *   - channel buffer: this one may be allocated to perform a synchronous recv,
+ *     or just preparing for the possibility of an instant response. The
+ *     response channel always allocates a buffer when entering process_stream,
+ *     which is immediately released if unused when leaving.
+ *   - buffer used by the mux sending side, often allocated by the mux's
+ *     snd_buf() handler to encode the outgoing channel's data.
+ *   - buffer permanently allocated at boot (e.g. temporary compression
+ *     buffers). If these fail, we can't boot.
+ */
+enum dynbuf_crit {
+	DB_GROW_RING = 0, // used to grow an existing buffer ring
+	DB_UNLIKELY,      // unlikely to be needed (e.g. L7 retries)
+	DB_MUX_RX,        // buffer used to store incoming data from the system
+	DB_SE_RX,         // buffer used to store incoming data for the channel
+	DB_CHANNEL,       // buffer used by the channel for synchronous reads
+	DB_MUX_TX,        // buffer used to store outgoing mux data
+	DB_PERMANENT,     // buffers permanently allocated.
+};
+
 /* an element of the <buffer_wq> list. It represents an object that need to
  * acquire a buffer to continue its process. */
 struct buffer_wait {

include/haproxy/dynbuf.h

@@ -63,10 +63,11 @@ static inline int buffer_almost_full(const struct buffer *buf)
  * performance. Due to the difficult buffer_wait management, they are not
  * subject to forced allocation failures either.
  */
-#define b_alloc(_buf) \
+#define b_alloc(_buf, _crit)			\
 ({						\
 	char *_area;				\
 	struct buffer *_retbuf = _buf;		\
+	enum dynbuf_crit _criticality __maybe_unused = _crit;	\
 						\
 	if (!_retbuf->size) {			\
 		*_retbuf = BUF_WANTED;					\

include/haproxy/quic_conn.h

@@ -138,7 +138,7 @@ static inline struct ncbuf *quic_get_ncbuf(struct ncbuf *ncbuf)
 	if (!ncb_is_null(ncbuf))
 		return ncbuf;
 
-	b_alloc(&buf);
+	b_alloc(&buf, DB_MUX_RX);
 	BUG_ON(b_is_null(&buf));
 
 	*ncbuf = ncb_make(buf.area, buf.size, 0);