Log large HTTP payloads in chunks locally and remotely¶

# Log POST request payloads remotely via HSL to a syslog server and locally.

# Based on Steve Hillier's example and the HTTP::collect wiki page
# https://clouddocs.f5.com/api/irules/http__collect.html
# Note that although any size payload can theoretically be collected, the maximum size of a Tcl variable in v9 and v10 is 4MB
#   with a smaller functional maximum after charset expansion of approximately 1Mb.
# In v11, the maximum variable size was increased to 32Mb.

when RULE_INIT {

    # Log debug to /var/log/ltm? 1=yes, 0=no
    set static::payload_dbg 1

    # Limit payload collection to 5Mb
    set static::max_collect_len 5368709

    # HSL pool name
    set static::hsl_pool "my_hsl_tcp_pool"

    # Max characters to log locally (must be less than 1024 bytes)
    # https://clouddocs.f5.com/api/irules/log.html
    set static::max_chars 900
}
when HTTP_REQUEST {

    # Only collect POST request payloads
    if {[HTTP::method] equals "POST"}{

        if {$static::payload_dbg}{log local0. "POST request"}

        # Open HSL connection
        set hsl [HSL::open -proto TCP -pool $static::hsl_pool]


        # Get the content length so we can request the data to be processed in the HTTP_REQUEST_DATA event.
        if {[HTTP::header exists "Content-Length"]}{
            set content_length [HTTP::header "Content-Length"]
        } else {
            set content_length 0
        }
        # content_length of 0 indicates chunked data (of unknown size)
        if {$content_length > 0 && $content_length < $static::max_collect_len}{
            set collect_length $content_length
        } else {
            set collect_length $static::max_collect_len
        }
        if {$static::payload_dbg}{log local0. "Content-Length: $content_length, Collect length: $collect_length"}
    }
}

when HTTP_REQUEST_DATA {

    # Log the bytes collected
    if {$static::payload_dbg}{log local0. "Collected [HTTP::payload length] bytes"}

    # Send all the collected payload to the remote syslog server
    HSL::send $hsl "<190>[HTTP::payload]\n"

    # Log the payload locally
    if {[HTTP::payload length] < $static::max_chars}{
        log local0. "Payload=[HTTP::payload]"
    } else {
        # Initialize variables
        set remaining $payload
        set position 0
        set count 1
        set bytes_logged 0

        # Loop through and log each chunk of the payload
        while {[string length $remaining] > $static::max_chars}{

            # Get the current chunk to log (subtract 1 from the end as string range is 0 indexed)
            set current [string range $remaining $position [expr {$position + $static::max_chars -1}]]
            log local0. "chunk $count=$current"

            # Add the length of the current chunk to the position for the next chunk
            incr position [string length $current]

            # Get the next chunk to log
            set remaining [string range $remaining $position end]
            incr count
            incr bytes_logged $position
            log local0. "remaining bytes=[string length $remaining], \$position=$position, \$count=$count, \$bytes_logged=$bytes_logged"
        }
        if {[string length $remaining]}{
            log local0. "chunk $count=$current"
            incr bytes_logged [string length $remaining]
        }
        log local0. "Logged $count chunks for a total of $bytes_logged bytes"
    }
}