diff --git a/.gitignore b/.gitignore index 32042ae0..2ee9ed4b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ rosetta .DS_Store share hooks/__pycache__/ - +site/ # wrangler files .wrangler .dev.vars* diff --git a/content/README.md b/content/README.md index 4652df82..02c35ceb 100644 --- a/content/README.md +++ b/content/README.md @@ -24,6 +24,7 @@ and benefit from the expertise shared in this repository. |Date|Headline| |---|---| +|2026-07-30|[Audit Log Forwarding to Rsyslog](cluster-configuration/logging/forwarding-demo/)| |2026-07-22|[Hosted Control Plane and Proxy](cluster-installation/hosted-control-plane/proxy/)| |2026-07-18|[Gateway API examples and ingress sharding](networking/gateway-api/)| |2026-07-17|[Gatekeeper — Automatic SCC Assignment](cluster-configuration/gatekeeper-opa/automatic-scc-assignment/)| diff --git a/content/cluster-configuration/logging/forwarding-demo/deploy-fluentd.yaml b/content/cluster-configuration/logging/forwarding-demo/deploy-fluentd.yaml deleted file mode 100644 index 63ea9f7f..00000000 --- a/content/cluster-configuration/logging/forwarding-demo/deploy-fluentd.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: fluentd-log -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi - storageClassName: managed-nfs-storage ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: fluentd -spec: - selector: - matchLabels: - app: fluentd - replicas: 1 - template: - metadata: - labels: - app: fluentd - spec: - containers: - - name: fluentd - image: docker.io/fluent/fluentd:v1.3-debian-1 - ports: - - containerPort: 24224 - volumeMounts: - - mountPath: /fluentd/log - name: fluentd-log - volumes: - - name: fluentd-log - persistentVolumeClaim: - claimName: fluentd-log ---- -apiVersion: v1 -kind: Service -metadata: - name: fluentd -spec: - selector: - app: fluentd - ports: - - protocol: TCP - port: 24224 - targetPort: 24224 diff --git a/content/cluster-configuration/logging/forwarding-demo/deploy-rsyslog.yaml b/content/cluster-configuration/logging/forwarding-demo/deploy-rsyslog.yaml new file mode 100644 index 00000000..a713aea2 --- /dev/null +++ b/content/cluster-configuration/logging/forwarding-demo/deploy-rsyslog.yaml @@ -0,0 +1,116 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rsyslog-config +data: + rsyslog.conf: | + global(workDirectory="/var/lib/rsyslog") + + module(load="imtcp") + input(type="imtcp" port="24224") + + template(name="RemoteLogFile" type="string" + string="/log/%HOSTNAME%/%PROGRAMNAME%.log") + + *.* action(type="omfile" file="/log/all.log") + *.* action(type="omfile" dynaFile="RemoteLogFile") +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: rsyslog-log +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rsyslog +spec: + selector: + matchLabels: + app: rsyslog + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: rsyslog + spec: + containers: + - name: rsyslog + image: registry.redhat.io/rhel9/rsyslog:9.8-1784815382 + ports: + - containerPort: 24224 + volumeMounts: + - mountPath: /log + name: rsyslog-log + - mountPath: /etc/rsyslog.conf + name: rsyslog-config + subPath: rsyslog.conf + - name: jq + image: registry.access.redhat.com/hi/jq:latest-builder + command: + - "/bin/sh" + - "-c" + - | + tail --quiet -f /log/*/kubeAPI.log | cut -d']' -f2- | jq -r ' + def pad(n): tostring | (n - length) as $p | (" " * ([0, $p] | max)) + .; + def to_epoch: + split(".") | + (.[0] + "Z" | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) as $secs | + ("0." + (.[1] | rtrimstr("Z")) | tonumber) as $frac | + $secs + $frac; + (.requestReceivedTimestamp | to_epoch) as $req | + (.stageTimestamp | to_epoch) as $stage | + ((($stage - $req) * 1000000 | round) / 1000) as $dur_ms | + (.requestReceivedTimestamp | split("T")[1] | split(".")[0]) as $time | + (.verb | ascii_upcase | pad(6)) as $verb | + ("\($dur_ms)ms" | pad(12)) as $dur | + "\($time) [\($verb)][\($dur)] [\(.responseStatus.code)] \(.requestURI) [\(.user.username)]" + ' + sleep infinity + volumeMounts: + - mountPath: /log + name: rsyslog-log + - name: cdt + image: registry.access.redhat.com/hi/go:latest-builder + command: + - "/bin/sh" + - "-c" + - | + export HOME=/go/ + git clone https://github.com/openshift/cluster-debug-tools.git + cd cluster-debug-tools/ + make + ./kubectl-dev_tool -h + sleep infinity + + volumeMounts: + - mountPath: /log + name: rsyslog-log + volumes: + - name: rsyslog-log + persistentVolumeClaim: + claimName: rsyslog-log + - name: rsyslog-config + configMap: + name: rsyslog-config +--- +apiVersion: v1 +kind: Service +metadata: + name: rsyslog +spec: + selector: + app: rsyslog + ports: + - protocol: TCP + port: 24224 + targetPort: 24224 diff --git a/content/cluster-configuration/logging/forwarding-demo/index.md b/content/cluster-configuration/logging/forwarding-demo/index.md index 1e626f4d..412a92ff 100644 --- a/content/cluster-configuration/logging/forwarding-demo/index.md +++ b/content/cluster-configuration/logging/forwarding-demo/index.md @@ -1,119 +1,159 @@ --- -title: Log Forwarding demo -linktitle: Log Forwarding API -weight: 18100 -description: TBD +title: Audit Log Forwarding to Rsyslog +linktitle: Audit Log Forwarding +description: Forward and filter OpenShift audit logs to a remote rsyslog server using ClusterLogForwarder +tags: ['logging', 'audit', 'v4.22'] --- -# Log Forwarding demo +# Audit Log Forwarding to Rsyslog -## Start fluentd to file logging +This example demonstrates how to forward Kubernetes API audit logs from an +OpenShift cluster to an external rsyslog server using the `ClusterLogForwarder` +resource. The filter is configured to capture only KubeVirt-related API +activity (`VirtualMachine`, `VirtualMachineInstance`, +`VirtualMachineInstanceMigration`), keeping the log volume focused and +manageable. -#### Create new project +Tested with: -=== "OC" +|Component|Version| +|---|---| +|OpenShift|v4.22.5| +|OpenShift Logging|v6.6.0| - ``` - oc new-project fluentd - ``` +Documentation: [Chapter 2. Configuring log forwarding](https://docs.redhat.com/en/documentation/red_hat_openshift_logging/6.6/html/configuring_logging/configuring-log-forwarding) -#### Deploy fluentd +## Start rsyslog to file logging + +First, deploy a simple rsyslog instance inside the cluster that receives +forwarded logs over TCP and writes them to persistent storage. + +### Create new project === "OC" + ```shell + oc new-project rsyslog ``` - oc apply -f {{ page.canonical_url }}deploy-fluentd.yaml + +### Deploy rsyslog + +=== "Download: deploy-rsyslog.yaml" + + ```shell + curl -L -O {{ page.canonical_url }}deploy-rsyslog.yaml ``` -=== "buildah-with-secret.yaml" +=== "deploy-rsyslog.yaml" ```yaml - --8<-- "content/cluster-configuration/logging/forwarding-demo/deploy-fluentd.yaml" + --8<-- "content/cluster-configuration/logging/forwarding-demo/deploy-rsyslog.yaml" + ``` + +=== "OC" + + ```shell + oc apply -f {{ page.canonical_url }}deploy-rsyslog.yaml ``` -### Test fluentd +The deployment includes three containers: + +- **rsyslog** — listens on TCP port 24224 and writes all received messages to `/log/all.log` as well as per-host/program files under `/log/`. +- **jq** — tails the KubeVirt audit log and pretty-prints each entry as a single line with verb, duration, status code, request URI, and user. +- **cdt** — builds [cluster-debug-tools](https://github.com/openshift/cluster-debug-tools) for additional troubleshooting. + +### Test rsyslog Login into the pod: -```bash -$ oc rsh deployment/fluentd bash +```shell +oc rsh deployment/rsyslog ``` -Run command inside the pod: -```bash -$ echo '{"message":"fooobar"}' | fluent-cat debug.log --host fluentd.fluentd.svc.cluster.local --port 24224 -$ grep fooobar /fluentd/log/data.log -2020-10-28T10:42:24+00:00 debug.log {"message":"fooobar"} +Send a test message and verify: + +```shell +logger --tcp --port 24224 --server localhost "test message from rsyslog" +cat /log/all.log ``` -## Deploy Cluster Logging +## Configure Cluster Logging - * Deploy OpenShift Logging Operator - * Deploy Elastic Search Operator from Red Hat. +Install the OpenShift Logging Operator from OperatorHub before proceeding. +The operator manages the `ClusterLogForwarder` custom resource used in the +next step. -```yaml -oc create -f - < audit.log - -$ kubectl dev_tool audit -f audit.log | head -14:31:55 [ WATCH][1h29m7.534747s] [200] /api/v1/namespaces/openshift-console-user-settings/configmaps?watch=true&fieldSelector=metadata.name%3Duser-settings-kubeadmin [kube:admin] -14:52:32 [ WATCH][1h8m30.575234s] [200] /apis/console.openshift.io/v1/consolenotifications?watch=true&resourceVersion=79435 [kube:admin] -15:02:13 [ WATCH][58m50.081621s] [200] /apis/console.openshift.io/v1/consolelinks?watch=true&resourceVersion=83292 [kube:admin] -15:08:26 [ WATCH][52m36.517786s] [200] /apis/apiregistration.k8s.io/v1/apiservices?watch=true&resourceVersion=85827 [kube:admin] -15:14:16 [ WATCH][46m47.349073s] [200] /apis/console.openshift.io/v1/consolequickstarts?watch=true&resourceVersion=88167 [kube:admin] -15:24:42 [ WATCH][36m20.76974s] [200] /apis/config.openshift.io/v1/clusterversions?watch=true&fieldSelector=metadata.name%3Dversion [kube:admin] -15:49:29 [ WATCH][9m51.001757s] [200] /api/v1/namespaces/openshift-kube-scheduler-operator/endpoints?allowWatchBookmarks=true&resourceVersion=102447&timeout=9m51s&timeoutSeconds=591&watch=true [system:serviceaccount:openshift-monitoring:prometheus-k8s] -15:49:30 [ WATCH][9m55.001174s] [200] /apis/config.openshift.io/v1/ingresses?allowWatchBookmarks=true&resourceVersion=102455&timeout=9m55s&timeoutSeconds=595&watch=true [system:serviceaccount:openshift-apiserver-operator:openshift-apiserver-operator] -15:49:37 [ WATCH][9m57.000757s] [200] /api/v1/namespaces/openshift-marketplace/pods?allowWatchBookmarks=true&resourceVersion=102513&timeout=9m57s&timeoutSeconds=597&watch=true [system:serviceaccount:openshift-monitoring:prometheus-k8s] -15:49:40 [ WATCH][9m50.001387s] [200] /api/v1/namespaces?allowWatchBookmarks=true&resourceVersion=102501&timeoutSeconds=590&watch=true [system:serviceaccount:openshift-monitoring:prometheus-operator] +```log +09:25:35 [CREATE][ 90.681ms] [201] /apis/kubevirt.io/v1/namespaces/rbohne-vms/virtualmachines [rbohne@redhat.com-admin] +09:26:30 [CREATE][ 67.071ms] [201] /apis/kubevirt.io/v1/namespaces/rbohne-vms/virtualmachineinstances [system:serviceaccount:openshift-cnv:kubevirt-controller] ``` - + diff --git a/mkdocs.yml b/mkdocs.yml index 5b9ca999..3a7fcc7c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -185,7 +185,7 @@ nav: - Telegram receiver: cluster-configuration/monitoring/alertmanager/telegram-receiver.md - Logging: - - Log Forwarding API: cluster-configuration/logging/forwarding-demo/index.md + - Audit Log Forwarding: cluster-configuration/logging/forwarding-demo/index.md - Image Registry: cluster-configuration/image-registry.md - MachineConfig: - cluster-configuration/machine-config/index.md