[Logging] Fix and standardize timestamp formats across Cloudwatch log configurations.

CHANGELOG.md

@@ -6,6 +6,9 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 3.15.0
 ------
 
+**CHANGES**
+- Fix and standardize timestamp formats in CloudWatch log configurations to facilitate maintainability and debugging.
+
 3.14.1
 ------
 

cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_config.json

@@ -2,10 +2,8 @@
   "timestamp_formats": {
     "month_first": "%b %-d %H:%M:%S",
     "default": "%Y-%m-%d %H:%M:%S,%f",
-    "bracket_default": "[%Y-%m-%d %H:%M:%S]",
-    "slurm": "%Y-%m-%dT%H:%M:%S.%f",
-    "chef": "[%Y-%m-%dT%H:%M:%S",
-    "json": ""
+    "default_seconds": "%Y-%m-%d %H:%M:%S",
+    "iso8610": "%Y-%m-%dT%H:%M:%S.%f"
   },
   "log_configs": [
     {
@@ -83,7 +81,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "chef",
+      "timestamp_format_key": "default",
       "file_path": "/var/log/chef-client.log",
       "log_stream_name": "chef-client",
       "schedulers": [
@@ -100,7 +98,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/bootstrap_error_msg",
       "log_stream_name": "bootstrap_error_msg",
       "schedulers": [
@@ -174,7 +172,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/clustermgtd.events",
       "log_stream_name": "clustermgtd_events",
       "schedulers": [
@@ -187,7 +185,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/slurm_resume.events",
       "log_stream_name": "slurm_resume_events",
       "schedulers": [
@@ -265,7 +263,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "slurm",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/slurmd.log",
       "log_stream_name": "slurmd",
       "schedulers": [
@@ -278,7 +276,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "slurm",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/slurmctld.log",
       "log_stream_name": "slurmctld",
       "schedulers": [
@@ -291,7 +289,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "slurm",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/slurmdbd.log",
       "log_stream_name": "slurmdbd",
       "schedulers": [
@@ -325,7 +323,7 @@
       ]
     },
     {
-      "timestamp_format_key": "default",
+      "timestamp_format_key": "default_seconds",
       "file_path": "/var/log/sssd/sssd.log",
       "log_stream_name": "sssd",
       "schedulers": [
@@ -346,7 +344,7 @@
       ]
     },
     {
-      "timestamp_format_key": "default",
+      "timestamp_format_key": "default_seconds",
       "file_path": "/var/log/sssd/sssd_default.log",
       "log_stream_name": "sssd_domain_default",
       "schedulers": [
@@ -389,7 +387,7 @@
       ]
     },
     {
-      "timestamp_format_key": "bracket_default",
+      "timestamp_format_key": "default",
       "file_path": "/var/log/parallelcluster/pcluster_dcv_connect.log",
       "log_stream_name": "dcv-ext-authenticator",
       "schedulers": [
@@ -522,7 +520,7 @@
       "feature_conditions": []
     },
     {
-      "timestamp_format_key": "json",
+      "timestamp_format_key": "iso8610",
       "file_path": "/var/log/parallelcluster/slurm_health_check.events",
       "log_stream_name": "slurm_health_check_events",
       "schedulers": [

cookbooks/aws-parallelcluster-environment/templates/directory_service/generate_ssh_key.sh.erb

@@ -1,4 +1,5 @@
 #!/bin/bash
+PS4="[$(date '+%Y-%m-%d %H:%M:%S,%3N')] "
 set -ex
 env
 

cookbooks/aws-parallelcluster-platform/files/dcv/pcluster_dcv_connect.sh

@@ -40,12 +40,6 @@ LOG_FILE_PATH="/var/log/parallelcluster/pcluster_dcv_connect.log"
 LOG_FILE_MAX_SIZE=5242880 # 5MB
 
 
-_fail() {
-  message=$1
-  >&2 echo "ERROR: ${message}"
-  exit 1
-}
-
 _validate_json() {
   json_param=$1
   message=$2
@@ -82,10 +76,16 @@ _log() {
     fi
 
     # append log
-    log_time=$(date "+%Y-%m-%d %H:%M:%S")
+    log_time=$(date "+%Y-%m-%d %H:%M:%S,%3N")
     echo "[${log_time}]: ${text}" >> "${LOG_FILE_PATH}"
 }
 
+_fail() {
+  message=$1
+  _log "ERROR: ${message}"
+  exit 1
+}
+
 _create_dcv_session() {
     dcv_session_file="$1"
     shared_folder_path="$2"

cookbooks/aws-parallelcluster-shared/libraries/log_formatter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+#
+# Custom log formatter with millisecond precision timestamps.
+# This formatter is used to standardize log output across all Chef runs
+# for improved log parsing and debugging capabilities.
+#
+class MillisecondFormatter < Mixlib::Log::Formatter
+  def call(severity, time, _progname, msg)
+    "[#{time.strftime('%Y-%m-%dT%H:%M:%S.%3N%z')}] #{severity}: #{msg}\n"
+  end
+end
+
+# Apply the formatter globally to Chef logs
+Chef::Log.logger.formatter = MillisecondFormatter.new