前言
本篇記錄如何在 K8S 透過 Filebeat 將 .Net Core 3.1 Web 專案的 Log 拋到 ELK上
.Net Core 3.1 NLog 專案設定
安裝 NLog 套件
dotnet add package NLog --version 4.7.15
dotnet add package NLog.Extensions.Logging --version 1.7.4
dotnet add package NLog.Web.AspNetCore --version 4.14.0
Program.cs
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using NLog.Extensions.Logging;
using NLog.Web;
namespace OctopusProject_Core_Sample
{
public class Program
{
public static void Main(string[] args)
{
CreateHostBuilder(args).Build().Run();
}
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.ConfigureLogging((hostContext, logging) =>
{
logging.ClearProviders();
logging.AddNLogWeb(new NLogLoggingConfiguration(hostContext.Configuration.GetSection("NLog")));
})
.ConfigureWebHostDefaults(webBuilder =>
{
webBuilder.UseStartup<Startup>();
});
}
}
appsettings.json
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"AllowedHosts": "*",
"AppSettings": {
"Env": "local"
},
"NLog": {
"throwConfigExceptions": true,
"targets": {
"async": true,
"logconsole": {
"type": "Console",
"layout": "${date:universalTime=true:format=yyyy-MM-dd HH\\:mm\\:ss.fff} ${threadid} ${level} ${logger} ${message}"
}
},
"rules": [
{
"logger": "*",
"minLevel": "Info",
"writeTo": "logconsole"
}
]
}
}
HomeController.cs 新增一支 API 做 Log 測試
public void TestLog()
{
_logger.LogInformation(".net core 3.1 log test");
}
專案部屬到 K8S 後 先測試 Log
確認 .net core 3.1 log test 是否正確輸出到 Console
# 呼叫一次 TestLog API
curl http://octopuscoresampledevelop.leozheng0512.com/Home/TestLog
# 查看 Pod
kubectl get pod -n web
# 輸出
NAME READY STATUS RESTARTS AGE
octopusproject-core-sample-674bf46d56-6bjj7 1/1 Running 0 39s
# 查看 Pod 的 Log
kubectl logs octopusproject-core-sample-674bf46d56-6bjj7 -n web
# 輸出
2022-06-04 09:13:40.830 1 Warn Microsoft.AspNetCore.DataProtection.Repositories.FileSystemXmlRepository Storing keys in a directory '/root/.aspnet/DataProtection-Keys' that may not be persisted outside of the container. Protected data will be unavailable when container is destroyed.
2022-06-04 09:13:41.102 1 Warn Microsoft.AspNetCore.DataProtection.KeyManagement.XmlKeyManager No XML encryptor configured. Key {5a24d768-96e6-4350-b756-750dc436c832} may be persisted to storage in unencrypted form.
2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Now listening on: http://[::]:80
2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Application started. Press Ctrl+C to shut down.
2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Hosting environment: develop
2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Content root path: /app
2022-06-04 09:13:41.759 9 Warn Microsoft.AspNetCore.HttpsPolicy.HttpsRedirectionMiddleware Failed to determine the https port for redirect.
2022-06-04 09:14:11.402 9 Info OctopusProject_Core_Sample.Controllers.HomeController .net core 3.1 log test
K8S Pod Console Log 路徑位置
cd /var/log/containers
ls
# 輸出
calico-node-wq2dd_kube-system_calico-node-2dd4fa9812098360b42584a1bd2dc7badef38b62fe16053c234939d453f1c6e6.log
calico-node-wq2dd_kube-system_calico-node-bf040192c9ec900555c6fbac4562acb3cad56a02bbab11269568dd4da6231e5b.log
calico-node-wq2dd_kube-system_flexvol-driver-27edce0f02bf7e49210bbd060305b7d0e361a191542025f6979e2fc6bcea817b.log
calico-node-wq2dd_kube-system_install-cni-4b3430decc9808b3ea9768e876522fb9ae9de82d8826fb8e10f94725519a63f5.log
calico-node-wq2dd_kube-system_upgrade-ipam-1a1d645f821f091182942deff1b36629cab2102f73cf03c993eac187e8b812bc.log
dockersample-webapi-7d66f7cdb7-57zgm_web-sample_web-container-6475e3ffd3f7b66669a7aa79ab8cf93e6a69ea15f08841bf68cd3d2900c13093.log
dockersample-webapi-7d66f7cdb7-57zgm_web-sample_web-container-f36673f4d78b9703d7a9e32c4e067928fb8cfeaf60b958925ba0626b14f072ab.log
filebeat-bz25z_elk_filebeat-4288643b977974e933286e91b27029f2125eb5fe65a389532ffea778df5db213.log
kube-proxy-4gmr8_kube-system_kube-proxy-62e43a2b6fee958efe8fe4bf8cbe4d539fdbdd288048d7c5f93a651f90b3e6ba.log
kube-proxy-4gmr8_kube-system_kube-proxy-81eb5b1ccc5c9ca517ffe8e11470c28f814534e49289db00397b6a72a05e30a3.log
myapp-deployment-6b8b598bf9-vcwsq_default_nginx-534d76d66a7a14889ba0d3b84e98bb987d57a3982e5e44a8973b20998b9eba8a.log
myapp-deployment-6b8b598bf9-vcwsq_default_nginx-b41242dffdc0e66323fa2de39868f8779f20f411d03a895f5deee642934078de.log
octopusproject-core-sample-674bf46d56-6bjj7_web_web-container-ee6763c1dbb6428c970c3e1291fef8896bccd2a54f35869b3b1159e5a93f664e.log
speaker-ppsbm_metallb-system_speaker-55afd6a07b10bb72e02c884582b38a4a74cd2586818e46cfbbb1d3e0c1e086a0.log
speaker-ppsbm_metallb-system_speaker-ef21c244326b9162a3a3fe17088d1e7148faa275140c52ced7a388522b6ca565.log
發現了 pod octopusproject-core-sample-674bf46d56-6bjj7 的 log 再進一步查看
cat octopusproject-core-sample-674bf46d56-6bjj7_web_web-container-ee6763c1dbb6428c970c3e1291fef8896bccd2a54f35869b3b1159e5a93f664e.log
# 輸出
{"log":"2022-06-04 09:13:40.830 1 Warn Microsoft.AspNetCore.DataProtection.Repositories.FileSystemXmlRepository Storing keys in a directory '/root/.aspnet/DataProtection-Keys' that may not be persisted outside of the container. Protected data will be unavailable when container is destroyed.\n","stream":"stdout","time":"2022-06-04T09:13:41.07036044Z"}
{"log":"2022-06-04 09:13:41.102 1 Warn Microsoft.AspNetCore.DataProtection.KeyManagement.XmlKeyManager No XML encryptor configured. Key {5a24d768-96e6-4350-b756-750dc436c832} may be persisted to storage in unencrypted form.\n","stream":"stdout","time":"2022-06-04T09:13:41.104297116Z"}
{"log":"2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Now listening on: http://[::]:80\n","stream":"stdout","time":"2022-06-04T09:13:41.264901406Z"}
{"log":"2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Application started. Press Ctrl+C to shut down.\n","stream":"stdout","time":"2022-06-04T09:13:41.265114295Z"}
{"log":"2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Hosting environment: develop\n","stream":"stdout","time":"2022-06-04T09:13:41.265261251Z"}
{"log":"2022-06-04 09:13:41.261 1 Info Microsoft.Hosting.Lifetime Content root path: /app\n","stream":"stdout","time":"2022-06-04T09:13:41.265403696Z"}
{"log":"2022-06-04 09:13:41.759 9 Warn Microsoft.AspNetCore.HttpsPolicy.HttpsRedirectionMiddleware Failed to determine the https port for redirect.\n","stream":"stdout","time":"2022-06-04T09:13:41.761472851Z"}
{"log":"2022-06-04 09:14:11.402 9 Info OctopusProject_Core_Sample.Controllers.HomeController .net core 3.1 log test\n","stream":"stdout","time":"2022-06-04T09:14:11.404309831Z"}
接下來 就是將 filebeat 抓的 Log 路徑 設在 /var/log/containers 用以轉拋
K8S 設定 filebeat
先建立 k8s-filebeat.yml 檔
exclude_files 可以排除檔案 (將一些不需要轉拋 Log 的檔案加以排除)
vi k8s-filebeat.yml
apiVersion: v1
kind: Namespace
metadata:
name: elk
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: elk
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
fields:
service: weblog
multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
multiline.negate: true
multiline.match: after
exclude_files: [ "calico", "dashboard", "ingress", "kube-proxy", "kubernetes", "metallb", "filebeat" ]
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
fields:
env: dev
output.redis:
enabled: true
hosts: ["x.x.x.x"]
key: log_test
password: xxx
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: elk
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.8.0
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0640
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
- name: data
hostPath:
# When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: elk
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: filebeat
namespace: elk
subjects:
- kind: ServiceAccount
name: filebeat
namespace: elk
roleRef:
kind: Role
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: filebeat-kubeadm-config
namespace: elk
subjects:
- kind: ServiceAccount
name: filebeat
namespace: elk
roleRef:
kind: Role
name: filebeat-kubeadm-config
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
- nodes
verbs:
- get
- watch
- list
- apiGroups: ["apps"]
resources:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: filebeat
# should be the namespace where filebeat is running
namespace: elk
labels:
k8s-app: filebeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: filebeat-kubeadm-config
namespace: elk
labels:
k8s-app: filebeat
rules:
- apiGroups: [""]
resources:
- configmaps
resourceNames:
- kubeadm-config
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: elk
labels:
k8s-app: filebeat
---
建立 k8s-filebeat 服務
kubectl create -f k8s-filebeat.yml
# 輸出
namespace/elk created
configmap/filebeat-config created
daemonset.apps/filebeat created
clusterrolebinding.rbac.authorization.k8s.io/filebeat created
rolebinding.rbac.authorization.k8s.io/filebeat created
rolebinding.rbac.authorization.k8s.io/filebeat-kubeadm-config created
clusterrole.rbac.authorization.k8s.io/filebeat created
role.rbac.authorization.k8s.io/filebeat created
role.rbac.authorization.k8s.io/filebeat-kubeadm-config created
serviceaccount/filebeat created
設定好後 即可查看 Kibana 是否有正確呈現 Log
轉載請註明來源，若有任何錯誤或表達不清楚的地方，歡迎在下方評論區留言，也可以來信至 leozheng0621@gmail.com
如果文章對您有幫助，歡迎斗內(donate)，請我喝杯咖啡
