rekahsoft: Proof-of-concept loki logging improvements and upgrade

* unguix/loki-home-rekahsoft-ca/rsyslog.conf: rsyslog configuration used for syslog aggreator

* unguix/loki-home-rekahsoft-ca/promtail-config.yaml (scrape_configs): Add syslog scape configuration

* unguix/loki-home-rekahsoft-ca/loki.sh: Add rsyslog which acts as an syslog aggregator in front of promtail

* unguix/loki-home-rekahsoft-ca/loki-config.yaml (ingester): Add 'wal' (required after upgrade)

unguix/loki-home-rekahsoft-ca/loki-config.yaml

@@ -14,6 +14,8 @@ ingester:
   chunk_idle_period: 5m
   chunk_retain_period: 30s
   max_transfer_retries: 0
+  wal:
+    dir: /tmp/wal
 
 schema_config:
   configs:

unguix/loki-home-rekahsoft-ca/loki.sh

@@ -2,12 +2,28 @@
 
 docker run -d \
   --restart unless-stopped \
+  --name loki \
   -v /var/lib/loki/config:/mnt/config \
   -p 3100:3100 \
-docker.nexus.home.rekahsoft.ca/grafana/loki:1.6.0 -config.file=/mnt/config/loki-config.yaml
+docker.nexus.home.rekahsoft.ca/grafana/loki:2.7.4 -config.file=/mnt/config/loki-config.yaml
 
 docker run -d \
   --restart unless-stopped \
+  --name rsyslog \
+  -v /var/lib/rsyslog/config:/config \
+  -v /var/lib/rsyslog/work:/work \
+  -v /var/lib/rsyslog/logs:/logs \
+  -p 514:514 \
+  -p 514:514/udp \
+  -e RSYSLOG_CONF=/config/rsyslog.conf \
+docker.nexus.home.rekahsoft.ca/rsyslog/syslog_appliance_alpine:8.36.0-3.7
+
+docker run -d \
+  --restart unless-stopped \
+  --name promtail \
   -v /var/lib/loki/config:/mnt/config \
   -v /var/log:/var/log \
-docker.nexus.home.rekahsoft.ca/grafana/promtail:1.6.0 -config.file=/mnt/config/promtail-config.yaml
+  -p 1514:1514 \
+  -p 1514:1514/udp \
+  -p 9080:9080 \
+docker.nexus.home.rekahsoft.ca/grafana/promtail:2.7.4 -config.file=/mnt/config/promtail-config.yaml

unguix/loki-home-rekahsoft-ca/promtail-config.yaml

@@ -16,3 +16,14 @@ scrape_configs:
     labels:
       job: varlogs
       __path__: /var/log/*log
+- job_name: syslog
+  syslog:
+    listen_address: 0.0.0.0:1514
+    listen_protocol: tcp
+    label_structured_data: yes
+    max_message_length: 1000
+    labels:
+      job: "syslog"
+  relabel_configs:
+    - source_labels: ['__syslog_message_hostname']
+      target_label: 'host'

unguix/loki-home-rekahsoft-ca/rsyslog.conf

@@ -0,0 +1,41 @@
+global(processInternalMessages="on")
+
+#module(load="imtcp" StreamDriver.AuthMode="anon" StreamDriver.Mode="1")
+module(load="impstats") # config.enabled=`echo $ENABLE_STATISTICS`)
+module(load="imrelp")
+module(load="imptcp")
+module(load="imudp" TimeRequery="500")
+
+module(load="omstdout")
+module(load="omelasticsearch")
+
+module(load="mmjsonparse")
+module(load="mmutf8fix")
+
+
+input(type="imptcp" port="514")
+input(type="imudp" port="514")
+input(type="imrelp" port="1601")
+
+template(name="log_to_files_dynafile" type="string" string=`echo $LOGFILES_STORE`)
+ruleset(name="log_to_files") {
+        /logs/debug;RSYSLOG_DebugFormat
+        action(type="omfile" dynafile="log_to_files_dynafile" name="log_to_logfiles")
+}
+
+ruleset(name="remote") {
+  # TODO: the target is a docker bridge ip (on the default bridge, so an alias cannot be used); this should be fixed
+  action(type="omfwd" Target="172.17.0.2" Port="1514" Protocol="tcp" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted")
+}
+
+#################### default ruleset begins ####################
+
+# we emit our own messages to docker console:
+syslog.* :omstdout:
+
+include(file="/config/droprules.conf" mode="optional")  # this permits the user to easily drop unwanted messages
+
+action(name="main_utf8fix" type="mmutf8fix" replacementChar="?")
+
+include(text=`echo $CNF_CALL_LOG_TO_LOGFILES`)
+call remote