feat: Issue 166 json formatted logs (#3376)

Co-authored-by: Alek Jouharyan <[email protected]>

api/app/settings/common.py

@@ -539,18 +539,23 @@
     with open(LOGGING_CONFIGURATION_FILE, "r") as f:
         LOGGING = json.loads(f.read())
 else:
+    LOG_FORMAT = env.str("LOG_FORMAT", default="generic")
     LOG_LEVEL = env.str("LOG_LEVEL", default="WARNING")
     LOGGING = {
         "version": 1,
         "disable_existing_loggers": False,
         "formatters": {
             "generic": {"format": "%(name)-12s %(levelname)-8s %(message)s"},
+            "json": {
+                "()": "util.logging.JsonFormatter",
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+            },
         },
         "handlers": {
             "console": {
                 "level": LOG_LEVEL,
                 "class": "logging.StreamHandler",
-                "formatter": "generic",
+                "formatter": LOG_FORMAT,
             }
         },
         "loggers": {

api/tests/unit/util/test_logging.py

@@ -0,0 +1,68 @@
+import json
+import logging
+
+from util.logging import JsonFormatter
+
+
+def test_json_formatter():
+    json_formatter = JsonFormatter()
+
+    log_record = logging.LogRecord(
+        name="test_logger",
+        level=logging.INFO,
+        pathname="test.py",
+        lineno=42,
+        msg="This is a test.",
+        args=(),
+        exc_info=None,
+    )
+    formatted_message = json_formatter.format(log_record)
+    json_message = json.loads(formatted_message)
+
+    assert "levelname" in json_message
+    assert "message" in json_message
+    assert "timestamp" in json_message
+    assert "logger_name" in json_message
+    assert "process_id" in json_message
+    assert "thread_name" in json_message
+
+
+def test_json_formatter_with_old_style_placeholders():
+    json_formatter = JsonFormatter()
+
+    log_record = logging.LogRecord(
+        name="test_logger",
+        level=logging.INFO,
+        pathname="example.py",
+        lineno=42,
+        msg="This is a test with old-style placeholders: %s and %s",
+        args=("arg1", "arg2"),
+        exc_info=None,
+    )
+
+    formatted_message = json_formatter.format(log_record)
+    parsed_json = json.loads(formatted_message)
+    assert (
+        parsed_json["message"]
+        == "This is a test with old-style placeholders: arg1 and arg2"
+    )
+
+
+def test_json_formatter_arguments_with_new_style_placeholders():
+    json_formatter = JsonFormatter()
+    log_record = logging.LogRecord(
+        name="test_logger",
+        level=logging.INFO,
+        pathname="example.py",
+        lineno=42,
+        msg="This is a test with new-style placeholders: {} and {}",
+        args=("arg1", "arg2"),
+        exc_info=None,
+    )
+
+    formatted_message = json_formatter.format(log_record)
+    parsed_json = json.loads(formatted_message)
+    assert (
+        parsed_json["message"]
+        == "This is a test with new-style placeholders: arg1 and arg2"
+    )

api/util/logging.py

@@ -0,0 +1,27 @@
+import json
+import logging
+
+
+class JsonFormatter(logging.Formatter):
+    """Custom formatter for json logs."""
+
+    def format(self, record):
+        """
+        %s is replaced with {} because legacy string formatting
+        conventions in django-axes module prevent correct
+        interpolation of arguments when using this formatter.
+        """
+        try:
+            log_message = record.msg.replace("%s", "{}")
+            formatted_message = log_message.format(*record.args)
+            log_record = {
+                "levelname": record.levelname,
+                "message": formatted_message,
+                "timestamp": self.formatTime(record, self.datefmt),
+                "logger_name": record.name,
+                "process_id": record.process,
+                "thread_name": record.threadName,
+            }
+            return json.dumps(log_record)
+        except (ValueError, TypeError) as e:
+            return f"Error formatting log record: {str(e)}"

docs/docs/deployment/hosting/locally-api.md

@@ -168,6 +168,7 @@ the below variables will be ignored.
   `django.core.management.utils.get_random_secret_key`. WARNING: If running multiple API instances, its vital that you
   define a shared DJANGO_SECRET_KEY.
 - `LOG_LEVEL`: DJANGO logging level. Can be one of `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`
+- `LOG_FORMAT`: Can be `generic` (plain-text) or `json`. Defaults to `generic`.
 - `ACCESS_LOG_LOCATION`: The location to store web logs generated by gunicorn if running as a Docker container. If not
   set, no logs will be stored. If set to `-` the logs will be sent to `stdout`.
 - `DJANGO_SETTINGS_MODULE`: python path to settings file for the given environment, e.g. "app.settings.develop"