fluentd-gke-config

1 개요

fluentd-gke-config

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-gke-config-v1.2.9
  namespace: kube-system
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
  google-fluentd.conf: |-
    @include config.d/*.conf
    # This match is placed before the all-matching output to provide metric
    # exporter with a process start timestamp for correct exporting of
    # cumulative metrics to Stackdriver.
    <match process_start>
      @type prometheus

      <metric>
        type gauge
        name process_start_time_seconds
        desc Timestamp of the process start in seconds
        key process_start_timestamp
      </metric>
    </match>

    # This filter allows to count the number of log entries read by fluentd
    # before they are processed by the output plugin. This in turn allows to
    # monitor the number of log entries that were read but never sent, e.g.
    # because of liveness probe removing buffer.
    <filter **>
      @type prometheus
      <metric>
        type counter
        name logging_entry_count
        desc Total number of log entries generated by either application containers or system components
      </metric>
    </filter>

    # This section is exclusive for k8s_container logs. These logs come with
    # 'stderr'/'stdout' tags.
    # We use a separate output stanza for 'k8s_node' logs with a smaller buffer
    # because node logs are less important than user's container logs.
    <match {stderr,stdout}>
      @type google_cloud

      # Try to detect JSON formatted log entries.
      detect_json true
      # Collect metrics in Prometheus registry about plugin activity.
      enable_monitoring true
      monitoring_type prometheus
      # Allow log entries from multiple containers to be sent in the same request.
      split_logs_by_tag false
      # Set the buffer type to file to improve the reliability and reduce the memory consumption
      buffer_type file
      buffer_path /var/run/google-fluentd/fluentd-gke-buffers/kubernetes.containers.buffer
      # Set queue_full action to block because we want to pause gracefully
      # in case of the off-the-limits load instead of throwing an exception
      buffer_queue_full_action block
      # Set the chunk limit conservatively to avoid exceeding the recommended
      # chunk size of 5MB per write request.
      buffer_chunk_limit 512k
      # Cap the combined memory usage of this buffer and the one below to
      # 512KiB/chunk * (6 + 2) chunks = 4 MiB
      buffer_queue_limit 6
      # Never wait more than 5 seconds before flushing logs in the non-error case.
      flush_interval 5s
      # Never wait longer than 30 seconds between retries.
      max_retry_wait 30
      # Disable the limit on the number of retries (retry forever).
      disable_retry_limit
      # Use multiple threads for processing.
      num_threads 2
      use_grpc true
      # Skip timestamp adjustment as this is in a controlled environment with
      # known timestamp format. This helps with CPU usage.
      adjust_invalid_timestamps false
    </match>



    # Attach local_resource_id for 'k8s_node' monitored resource.
    <filter **>
      @type record_transformer
      enable_ruby true
      <record>
        "logging.googleapis.com/local_resource_id" ${"k8s_node.#{ENV['NODE_NAME']}"}
      </record>
    </filter>

    <filter {docker,container-runtime,kubelet,kubelet-monitor,node-problem-detector,kube-container-runtime-monitor,node-journal}>
      @type record_modifier
      enable_ruby true
      <record>
        # Set a "gke.googleapis.com/log_type" label with value "system" under
        # "logging.googleapis.com/labels" for non-containerized systemd logs.
        _dummy_ ${if record["logging.googleapis.com/labels"].nil?; then; record["logging.googleapis.com/labels"] = Hash.new; end; record["logging.googleapis.com/labels"].merge!({ "gke.googleapis.com/log_type" => "system" }); nil}
      </record>
      # Delete this dummy field.
      remove_keys _dummy_
    </filter>

    # This section is exclusive for 'k8s_node' logs. These logs come with tags
    # that are neither 'stderr' or 'stdout'.
    # We use a separate output stanza for 'k8s_container' logs with a larger
    # buffer because user's container logs are more important than node logs.
    <match **>
      @type google_cloud

      detect_json true
      enable_monitoring true
      monitoring_type prometheus
      # Allow entries from multiple system logs to be sent in the same request.
      split_logs_by_tag false
      detect_subservice false
      buffer_type file
      buffer_path /var/run/google-fluentd/fluentd-gke-buffers/kubernetes.system.buffer
      buffer_queue_full_action block
      buffer_chunk_limit 512k
      buffer_queue_limit 2
      flush_interval 5s
      max_retry_wait 30
      disable_retry_limit
      num_threads 2
      use_grpc true
      # Skip timestamp adjustment as this is in a controlled environment with
      # known timestamp format. This helps with CPU usage.
      adjust_invalid_timestamps false
    </match>

2 같이 보기

• fluentd-gcp-config

3 참고

• https://cloud.google.com/logging/docs/agent/configuration