Docker Log Observability: Analyzing Container Logs in HashiCorp Nomad with Vector, Loki, and Grafana

# Add the repository to Apt sources:
echo
“deb [arch=$(dpkg –print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu
$(. /etc/os-release && echo “$VERSION_CODENAME“) stable” |
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update

server {
enabled = true
bootstrap_expect = 1
search {
fuzzy_enabled = true
limit_query = 200
limit_results = 1000
min_term_length = 5
}
}

# Enable the client
client {
enabled = true
options {
“driver.raw_exec.enable” = “1”
“docker.privileged.enabled” = “true”
}

server_join {
retry_join = [ “127.0.0.1” ]
}

}

plugin “docker” {
config {
endpoint = “unix:///var/run/docker.sock”

extra_labels = [“job_name”, “job_id”, “task_group_name”, “task_name”, “namespace”, “node_name”, “node_id”]

volumes {
enabled = true
selinuxlabel = “z”
}

allow_privileged = true

}
}

telemetry {
collection_interval = “15s”
disable_hostname = true
prometheus_metrics = true
publish_allocation_metrics = true
publish_node_metrics = true
}

==> Nomad agent configuration:

Advertise Addrs: HTTP: 192.168.1.32:4646; RPC: 192.168.1.32:4647; Serf: 192.168.1.32:4648
Bind Addrs: HTTP: [0.0.0.0:4646]; RPC: 0.0.0.0:4647; Serf: 0.0.0.0:4648
Client: true
Log Level: INFO
Node Id: 2921dae9-99dc-a65d-1a1f-25d9822c1500
Region: global (DC: dc1)
Server: true
Version: 1.7.7

==> Nomad agent started! Log data will stream in below:

group “loki” {
count = 1

network {
mode = “host”
port “loki” {
to = 3100
static = 3100
}
}

service {
name = “loki”
port = “loki”
provider = “nomad”
}

task “loki” {
driver = “docker”
user = “root”
config {
image = “grafana/loki:2.9.7”
args = [
“-config.file”,
“local/config.yml”,
]
volumes = [“/loki_data:/loki”]
ports = [“loki”]
}
template {
data=<<EOH
auth_enabled: false
server:
http_listen_port: 3100
ingester:
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
# Any chunk not receiving new logs in this time will be flushed
chunk_idle_period: 1h
# All chunks will be flushed when they hit this age, default is 1h
max_chunk_age: 1h
# Loki will attempt to build chunks up to 1.5MB, flushing if chunk_idle_period or max_chunk_age is reached first
chunk_target_size: 1048576
# Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
chunk_retain_period: 30s
max_transfer_retries: 0 # Chunk transfers disabled
wal:
enabled: true
dir: “/loki/wal”
schema_config:
configs:
– from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /loki/boltdb-shipper-active
cache_location: /loki/boltdb-shipper-cache
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
shared_store: filesystem
filesystem:
directory: /loki/chunks
compactor:
working_directory: /loki/boltdb-shipper-compactor
shared_store: filesystem
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
EOH
destination = “local/config.yml”
change_mode = “restart”
}
resources {
cpu = 1000 #Mhz
memory = 1000 #MB
}
}
}
}

type = “service”

group “logger” {
count = 1

task “logger” {

driver = “docker”

config {
image = “chentex/random-logger:latest”
}

resources {
cpu = 100 # 100 MHz
memory = 100 # 100MB
}

}
}
}

group “vector” {
count = 1

network {
port “api” {
to = 8686
}
}

ephemeral_disk {
size = 500
sticky = true
}

task “vector” {
driver = “docker”
config {
image = “timberio/vector:0.30.0-debian”
ports = [“api”]
volumes = [“/var/run/docker.sock:/var/run/docker.sock”]
}
env {
VECTOR_CONFIG = “local/vector.toml”
VECTOR_REQUIRE_HEALTHY = “false”
}
resources {
cpu = 100 # 100 MHz
memory = 100 # 100MB
}
# template with Vector’s configuration
template {
destination = “local/vector.toml”
change_mode = “signal”
change_signal = “SIGHUP”
# overriding the delimiters to [[ ]] to avoid conflicts with Vector’s native templating, which also uses {{ }}
left_delimiter = “[[“
right_delimiter = “]]”
data=<<EOH
data_dir = “alloc/data/vector/”
[api]
enabled = true
address = “0.0.0.0:8686”
playground = true
[sources.logs]
type = “docker_logs”
[sinks.out]
type = “console”
inputs = [ “logs” ]
encoding.codec = “json”
target = “stdout”
[sinks.loki]
type = “loki”
compression = “snappy”
encoding.codec = “json”
inputs = [“logs”]
endpoint = “http://[[ range nomadService “loki” ]][[.Address]]:[[.Port]][[ end ]]”
healthcheck.enabled = true
out_of_order_action = “drop”
# remove fields that have been converted to labels to avoid having the field twice
remove_label_fields = true
[sinks.loki.labels]
# See https://vector.dev/docs/reference/vrl/expressions/#path-example-nested-path
job = “{{label.”com.hashicorp.nomad.job_name” }}”
task = “{{label.”com.hashicorp.nomad.task_name” }}”
group = “{{label.”com.hashicorp.nomad.task_group_name” }}”
namespace = “{{label.”com.hashicorp.nomad.namespace” }}”
node = “{{label.”com.hashicorp.nomad.node_name” }}”
EOH
}
kill_timeout = “30s”
}
}
}

group “grafana” {
count = 1

network {
mode = “host”

port “grafana” {
to = 3000
static = 3000
}
}

task “grafana” {
driver = “docker”

env {
GF_LOG_LEVEL = “ERROR”
GF_LOG_MODE = “console”
GF_PATHS_DATA = “/var/lib/grafana”
}

user = “root”

config {
image = “grafana/grafana:10.4.2”
ports = [“grafana”]
volumes = [“/grafana_volume:/var/lib/grafana”]
}

resources {
cpu = 2000
memory = 2000
}

}
}

}