trafficserver: Caching doesn't seem to work

We’ve setup a ats instance that has both port 80 and 443 (one instance). We’ve configured it to connect to our mediawiki backends that run nginx. We are finding that it’s not caching. Or at least sometimes caching intermittently.

Our records.config:

# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.config.en.html
# This file is managed by Puppet.

# traffic_manager TCP ports
CONFIG proxy.config.http.server_ports STRING 443:ssl 443:ipv6:ssl 80 80:ipv6
CONFIG proxy.config.admin.user_id STRING trafficserver

CONFIG proxy.config.http.insert_request_via_str INT 3
CONFIG proxy.config.http.insert_response_via_str INT 3
CONFIG proxy.config.http.insert_client_ip INT 0
CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 0
CONFIG proxy.config.http.response_server_enabled INT 2

CONFIG proxy.config.url_remap.pristine_host_hdr INT 1
CONFIG proxy.config.disable_configuration_modification INT 1

CONFIG proxy.config.plugin.lua.max_states INT 256

# Network settings
CONFIG proxy.config.net.connections_throttle INT 0
CONFIG proxy.config.net.sock_option_flag_in INT 5
CONFIG proxy.config.net.default_inactivity_timeout INT 360
CONFIG proxy.config.net.max_connections_in INT 270000
CONFIG proxy.config.net.max_connections_active_in INT 90000
# HTTP/2 settings
CONFIG proxy.config.http2.stream_priority_enabled INT 0
CONFIG proxy.config.http2.max_settings_per_frame INT 7
CONFIG proxy.config.http2.max_settings_per_minute INT 14
CONFIG proxy.config.http2.max_settings_frames_per_minute INT 14
CONFIG proxy.config.http2.max_ping_frames_per_minute INT 60
CONFIG proxy.config.http2.max_priority_frames_per_minute INT 0
CONFIG proxy.config.http2.min_avg_window_update FLOAT 2560.0
CONFIG proxy.config.http2.accept_no_activity_timeout INT 180
CONFIG proxy.config.http2.no_activity_timeout_in INT 180
CONFIG proxy.config.http2.stream_error_rate_threshold FLOAT 1.0

# HTTP settings
CONFIG proxy.config.http.connect_attempts_timeout INT 9
CONFIG proxy.config.http.post_connect_attempts_timeout INT 11
CONFIG proxy.config.http.transaction_no_activity_timeout_in INT 200
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 180
CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
CONFIG proxy.config.http.accept_no_activity_timeout INT 180
CONFIG proxy.config.http.send_100_continue_response INT 0
CONFIG proxy.config.http.max_post_size INT 0
CONFIG proxy.config.http.allow_half_open INT 1
# keep-alive configuration for connections against origin servers
CONFIG proxy.config.http.keep_alive_enabled_out INT 1
CONFIG proxy.config.http.keep_alive_post_out INT 1

# Origin server connection reuse
CONFIG proxy.config.http.server_session_sharing.match STRING both
CONFIG proxy.config.http.server_session_sharing.pool STRING thread
# Allow origin server connection reuse for requests with Authorization
CONFIG proxy.config.http.auth_server_session_private INT 0

CONFIG proxy.config.http.cache.cache_responses_to_cookies INT 1
CONFIG proxy.config.http.cache.guaranteed_max_lifetime INT 86400
# Do not accept inbound connections until the cache is enabled
CONFIG proxy.config.http.wait_for_cache INT 2

#proxy.config.http.cache.when_to_revalidate INT 4

# RAM Cache
CONFIG proxy.config.cache.ram_cache.size INT -1
CONFIG proxy.config.cache.ram_cache_cutoff INT 8388608 # 8MB
# 0=no compression, 1=Fastlz, 2=Libz, 3=Liblzma
CONFIG proxy.config.cache.ram_cache.compress INT 2

# Disk Cache
# max_doc_size needs to be disabled in order for Read While Writer to function
# properly. See cache-basics.en.html#read-while-writer. We use Lua to avoid
# caching documents with explicit Content-Length > 1G, see
# do_global_read_response().
CONFIG proxy.config.cache.max_doc_size INT 0

# Parent proxy
CONFIG proxy.config.http.parent_proxy_routing_enable INT 1
CONFIG proxy.config.http.parent_proxy.self_detect INT 0
CONFIG proxy.config.http.parent_proxy.retry_time INT 1
CONFIG proxy.config.http.no_dns_just_forward_to_parent INT 0
CONFIG proxy.config.http.parent_proxy.connect_attempts_timeout INT 9

# XDebug Plugin
CONFIG proxy.config.diags.debug.enabled INT 1
#CONFIG proxy.config.diags.debug.tags STRING http|dns
CONFIG proxy.config.diags.debug.tags STRING http|cache

# Request coalescing default values:
# proxy.config.cache.enable_read_while_writer 1
# proxy.config.http.cache.max_open_read_retries -1
# proxy.config.http.cache.max_open_write_retries 1
# proxy.config.http.cache.open_write_fail_action 0
CONFIG proxy.config.cache.enable_read_while_writer INT 1
CONFIG proxy.config.http.cache.max_open_read_retries INT 50
CONFIG proxy.config.http.cache.max_open_write_retries INT 150
# In any case, fail open
CONFIG proxy.config.http.cache.open_write_fail_action INT 0

# logging: send diags to stdout/stderr (systemd journal)
CONFIG proxy.config.diags.output.status STRING O
CONFIG proxy.config.diags.output.note STRING O
CONFIG proxy.config.diags.output.warning STRING O
CONFIG proxy.config.diags.output.error STRING E
CONFIG proxy.config.diags.output.fatal STRING E
CONFIG proxy.config.diags.output.alert STRING E
CONFIG proxy.config.diags.output.emergency STRING E
# ATS stops logging if the logging directory has less than
# max_space_mb_headroom (default: 1000 MB) available
CONFIG proxy.config.log.max_space_mb_headroom INT 0
# Do not rotate logfiles
CONFIG proxy.config.log.rolling_enabled INT 0
# Flush logs immediately
CONFIG proxy.config.log.max_secs_per_buffer INT 0
# React immediatelly to log files being rotated
CONFIG proxy.config.log.file_stat_frequency INT 1
# Bumped from the default (9216) to address the following error: "Skipping the
# current log entry because its size exceeds the maximum payload space in a log
# buffer"
CONFIG proxy.config.log.log_buffer_size INT 147456
CONFIG proxy.config.log.max_line_size INT 147456

# Incoming TLS settings
CONFIG proxy.config.ssl.server.honor_cipher_order INT 1
CONFIG proxy.config.ssl.TLSv1 INT 0
CONFIG proxy.config.ssl.TLSv1_1 INT 0
CONFIG proxy.config.ssl.TLSv1_2 INT 1
CONFIG proxy.config.ssl.TLSv1_3 INT 1
CONFIG proxy.config.ssl.server.cipher_suite STRING -ALL:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256
CONFIG proxy.config.ssl.server.TLSv1_3.cipher_suites STRING TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256
CONFIG proxy.config.ssl.server.groups_list STRING X25519:prime256v1
CONFIG proxy.config.ssl.server.cert.path STRING /etc/ssl/localcerts/
CONFIG proxy.config.ssl.server.private_key.path STRING /etc/ssl/private/
CONFIG proxy.config.ssl.max_record_size INT 16383
CONFIG proxy.config.ssl.ocsp.enabled INT 0
CONFIG proxy.config.ssl.session_cache INT 2
CONFIG proxy.config.ssl.session_cache.timeout INT 86400
CONFIG proxy.config.ssl.session_cache.auto_clear INT 1
CONFIG proxy.config.ssl.session_cache.size INT 512000
CONFIG proxy.config.ssl.session_cache.num_buckets INT 32768
CONFIG proxy.config.ssl.cert.load_elevated INT 1
CONFIG proxy.config.ssl.server.session_ticket.enable INT 0
CONFIG proxy.config.ssl.handshake_timeout_in INT 60
CONFIG proxy.config.ssl.server.prioritize_chacha INT 1
# Outbound TLS settings
CONFIG proxy.config.ssl.client.TLSv1 INT 0
CONFIG proxy.config.ssl.client.TLSv1_1 INT 0
CONFIG proxy.config.ssl.client.TLSv1_2 INT 1
CONFIG proxy.config.ssl.client.TLSv1_3 INT 1
CONFIG proxy.config.ssl.client.cipher_suite STRING -ALL:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384
CONFIG proxy.config.ssl.client.TLSv1_3.cipher_suites STRING TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
# Origin server certificate validation settings
CONFIG proxy.config.ssl.client.verify.server INT 2
CONFIG proxy.config.ssl.client.CA.cert.path STRING /etc/ssl/certs
CONFIG proxy.config.ssl.client.CA.cert.filename STRING ca-certificates.crt

# Location of HTML template for error pages
CONFIG proxy.config.body_factory.template_sets_dir STRING /etc/trafficserver/error_template

CONFIG proxy.config.http.cache.required_headers INT 1

Our parent.config

# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/parent.config.en.html
# This file is managed by Puppet.
dest_domain=backends.miraheze.org parent="mw8.miraheze.org:443,mw9.miraheze.org:443,mw10.miraheze.org:443,mw11.miraheze.org:443" parent_is_proxy=false round_robin=strict

Our remap.config

....
map / https://backends.miraheze.org

Do we need a ats tls terminator and a ats backend for caching to work or is there something we are doing wrong in the config?

Here’s the response for one site from the backend:

HTTP/2 200 
server: nginx
date: Fri, 02 Apr 2021 17:50:13 GMT
content-type: text/html; charset=UTF-8
x-powered-by: PHP/7.3.27-1~deb10u1
x-content-type-options: nosniff
p3p: CP="This is not a P3P policy! See https://allthetropes.org/wiki/Special:CentralAutoLogin/P3P for more info."
content-language: en
vary: Accept-Encoding, Cookie, Authorization
cache-control: s-maxage=18000, must-revalidate, max-age=0
last-modified: Fri, 02 Apr 2021 12:50:13 GMT
x-request-id: bf43dd40382ddb8920728ec8
strict-transport-security: max-age=604800
x-served-by: mw8.miraheze.org
x-xss-protection: 1; mode=block
x-frame-options: ALLOW-FROM static.miraheze.org
age: 0
via: https/1.1 localhost (ApacheTrafficServer/8.0.2 [uScMsSf pSeN:t cCMpSs ])

we added some debug so via shows.