diff --git a/api/Makefile b/api/Makefile index ee8c91850d4c..da2496478537 100644 --- a/api/Makefile +++ b/api/Makefile @@ -83,7 +83,10 @@ django-collect-static: .PHONY: serve serve: - poetry run gunicorn --bind 0.0.0.0:8000 app.wsgi --reload + poetry run gunicorn --bind 0.0.0.0:8000 \ + --logger-class ${GUNICORN_LOGGER_CLASS:-'util.logging.GunicornJsonCapableLogger'} \ + --reload \ + app.wsgi .PHONY: generate-ld-client-types generate-ld-client-types: diff --git a/api/scripts/run-docker.sh b/api/scripts/run-docker.sh index 79133d4254a1..e1f603e53f13 100755 --- a/api/scripts/run-docker.sh +++ b/api/scripts/run-docker.sh @@ -22,7 +22,8 @@ function serve() { --workers ${GUNICORN_WORKERS:-3} \ --threads ${GUNICORN_THREADS:-2} \ --access-logfile $ACCESS_LOG_LOCATION \ - --access-logformat '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %({origin}i)s %({access-control-allow-origin}o)s' \ + --logger-class ${GUNICORN_LOGGER_CLASS:-'util.logging.GunicornJsonCapableLogger'} \ + --access-logformat ${ACCESS_LOG_FORMAT:-'%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %({origin}i)s %({access-control-allow-origin}o)s'} \ --keep-alive ${GUNICORN_KEEP_ALIVE:-2} \ ${STATSD_HOST:+--statsd-host $STATSD_HOST:$STATSD_PORT} \ ${STATSD_HOST:+--statsd-prefix $STATSD_PREFIX} \ diff --git a/api/tests/unit/util/test_logging.py b/api/tests/unit/util/test_logging.py index 296c3997148e..2e85a7eb2f10 100644 --- a/api/tests/unit/util/test_logging.py +++ b/api/tests/unit/util/test_logging.py @@ -4,7 +4,7 @@ from util.logging import JsonFormatter -def test_json_formatter(): +def test_json_formatter__outputs_expected(): json_formatter = JsonFormatter() log_record = logging.LogRecord( @@ -12,8 +12,8 @@ def test_json_formatter(): level=logging.INFO, pathname="test.py", lineno=42, - msg="This is a test.", - args=(), + msg="This is a test message with args: %s and %s", + args=("arg1", "arg2"), exc_info=None, ) formatted_message = json_formatter.format(log_record) @@ -25,44 +25,4 @@ def test_json_formatter(): assert "logger_name" in json_message assert "process_id" in json_message assert "thread_name" in json_message - - -def test_json_formatter_with_old_style_placeholders(): - json_formatter = JsonFormatter() - - log_record = logging.LogRecord( - name="test_logger", - level=logging.INFO, - pathname="example.py", - lineno=42, - msg="This is a test with old-style placeholders: %s and %s", - args=("arg1", "arg2"), - exc_info=None, - ) - - formatted_message = json_formatter.format(log_record) - parsed_json = json.loads(formatted_message) - assert ( - parsed_json["message"] - == "This is a test with old-style placeholders: arg1 and arg2" - ) - - -def test_json_formatter_arguments_with_new_style_placeholders(): - json_formatter = JsonFormatter() - log_record = logging.LogRecord( - name="test_logger", - level=logging.INFO, - pathname="example.py", - lineno=42, - msg="This is a test with new-style placeholders: {} and {}", - args=("arg1", "arg2"), - exc_info=None, - ) - - formatted_message = json_formatter.format(log_record) - parsed_json = json.loads(formatted_message) - assert ( - parsed_json["message"] - == "This is a test with new-style placeholders: arg1 and arg2" - ) + assert json_message["message"] == "This is a test message with args: arg1 and arg2" diff --git a/api/util/logging.py b/api/util/logging.py index b274f92b449b..32a81d08b2e1 100644 --- a/api/util/logging.py +++ b/api/util/logging.py @@ -1,27 +1,68 @@ import json import logging +import sys +from datetime import datetime +from typing import Any + +from django.conf import settings +from gunicorn.config import Config +from gunicorn.glogging import Logger as GunicornLogger class JsonFormatter(logging.Formatter): """Custom formatter for json logs.""" - def format(self, record): - """ - %s is replaced with {} because legacy string formatting - conventions in django-axes module prevent correct - interpolation of arguments when using this formatter. - """ + def get_json_record(self, record: logging.LogRecord) -> dict[str, Any]: + formatted_message = record.getMessage() + return { + "levelname": record.levelname, + "message": formatted_message, + "timestamp": self.formatTime(record, self.datefmt), + "logger_name": record.name, + "process_id": record.process, + "thread_name": record.threadName, + } + + def format(self, record: logging.LogRecord) -> str: try: - log_message = record.msg.replace("%s", "{}") - formatted_message = log_message.format(*record.args) - log_record = { - "levelname": record.levelname, - "message": formatted_message, - "timestamp": self.formatTime(record, self.datefmt), - "logger_name": record.name, - "process_id": record.process, - "thread_name": record.threadName, - } - return json.dumps(log_record) + return json.dumps(self.get_json_record(record)) except (ValueError, TypeError) as e: - return f"Error formatting log record: {str(e)}" + return json.dumps({"message": f"{e} when dumping log"}) + + +class GunicornAccessLogJsonFormatter(JsonFormatter): + def get_json_record(self, record: logging.LogRecord) -> dict[str, Any]: + response_time = datetime.strptime(record.args["t"], "[%d/%b/%Y:%H:%M:%S %z]") + url = record.args["U"] + if record.args["q"]: + url += f"?{record.args['q']}" + + return { + **super().get_json_record(record), + "time": response_time.isoformat(), + "path": url, + "remote_ip": record.args["h"], + "method": record.args["m"], + "status": str(record.args["s"]), + "user_agent": record.args["a"], + "referer": record.args["f"], + "duration_in_ms": record.args["M"], + "pid": record.args["p"], + } + + +class GunicornJsonCapableLogger(GunicornLogger): + def setup(self, cfg: Config) -> None: + super().setup(cfg) + if settings.LOG_FORMAT == "json": + self._set_handler( + self.error_log, + cfg.errorlog, + JsonFormatter(), + ) + self._set_handler( + self.access_log, + cfg.accesslog, + GunicornAccessLogJsonFormatter(), + stream=sys.stdout, + ) diff --git a/docs/docs/deployment/hosting/docker.md b/docs/docs/deployment/hosting/docker.md index 18ce7df565c5..0f6ed9d88871 100644 --- a/docs/docs/deployment/hosting/docker.md +++ b/docs/docs/deployment/hosting/docker.md @@ -20,13 +20,17 @@ new account at [http://localhost:8000/signup](http://localhost:8000/signup) As well as the Environment Variables specified in the [API](/deployment/hosting/locally-api#environment-variables) and [Front End](/deployment/hosting/locally-frontend#environment-variables) you can also specify the following: +- `GUNICORN_CMD_ARGS`: Gunicorn command line arguments. Overrides Flagsmith's defaults. See + [Gunicorn documentation](https://docs.gunicorn.org/en/stable/settings.html) for reference. - `GUNICORN_WORKERS`: The number of [Gunicorn Workers](https://docs.gunicorn.org/en/stable/settings.html#workers) that are created - `GUNICORN_THREADS`: The number of [Gunicorn Threads per Worker](https://docs.gunicorn.org/en/stable/settings.html#threads) - `GUNICORN_TIMEOUT`: The number of seconds before the [Gunicorn times out](https://docs.gunicorn.org/en/stable/settings.html#timeout) -- `ACCESS_LOG_LOCATION`: The location to write access logs to +- `ACCESS_LOG_FORMAT`: Message format for Gunicorn's access log. See + [variable details](https://docs.gunicorn.org/en/stable/settings.html#access-log-format) to define your own format. +- `ACCESS_LOG_LOCATION`: The location to write access logs to. If set to `-`, the logs will be sent to `stdout` ## Platform Architectures diff --git a/docs/docs/deployment/hosting/locally-api.md b/docs/docs/deployment/hosting/locally-api.md index b5f28bcd8cb8..cccd7f9341de 100644 --- a/docs/docs/deployment/hosting/locally-api.md +++ b/docs/docs/deployment/hosting/locally-api.md @@ -150,8 +150,12 @@ the below variables will be ignored. define a shared DJANGO_SECRET_KEY. - `LOG_LEVEL`: DJANGO logging level. Can be one of `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` - `LOG_FORMAT`: Can be `generic` (plain-text) or `json`. Defaults to `generic`. -- `ACCESS_LOG_LOCATION`: The location to store web logs generated by gunicorn if running as a Docker container. If not - set, no logs will be stored. If set to `-` the logs will be sent to `stdout`. +- `GUNICORN_CMD_ARGS`: Gunicorn command line arguments. Overrides Flagsmith's defaults. See + [Gunicorn documentation](https://docs.gunicorn.org/en/stable/settings.html) for reference. +- `ACCESS_LOG_FORMAT`: Message format for Gunicorn's access log. See + [variable details](https://docs.gunicorn.org/en/stable/settings.html#access-log-format) to define your own format. +- `ACCESS_LOG_LOCATION`: The location to store web logs generated by Gunicorn if running as a Docker container. If not + set, no logs will be stored. If set to `-`, the logs will be sent to `stdout`. - `DJANGO_SETTINGS_MODULE`: python path to settings file for the given environment, e.g. "app.settings.develop" - `ALLOW_ADMIN_INITIATION_VIA_CLI`: Enables the `bootstrap` management command which creates default admin user, organisation, and project.