Skip to content

Commit bf2373b

Browse files
committed
Merge branch 'master' into master-github
2 parents ae57682 + d8f73a2 commit bf2373b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+1756
-679
lines changed

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@ checkstyle.txt
1212
.env
1313
.direnv
1414
.envrc
15+
.elasticbeanstalk/

.gitlab-ci.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ deploydevelop:
2525
- develop
2626

2727
deployawsstaging:
28-
image: twstuart/elasticbeanstalk-pipenv
28+
image: bullettrain/elasticbeanstalk-pipenv
2929
stage: deploy-aws
3030
script:
3131
- export AWS_ACCESS_KEY_ID=$AWS_STAGING_ACCESS_KEY_ID
@@ -45,7 +45,7 @@ deployawsstaging:
4545
- staging
4646

4747
deployawsmaster:
48-
image: twstuart/elasticbeanstalk-pipenv
48+
image: bullettrain/elasticbeanstalk-pipenv
4949
stage: deploy-aws
5050
script:
5151
- export DATABASE_URL=$DATABASE_URL_PRODUCTION

Pipfile

+2-3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ pylint = "*"
1616
"autopep8" = "*"
1717
pytest = "*"
1818
pytest-django = "*"
19+
django-test-migrations = "*"
1920

2021
[packages]
2122
appdirs = "*"
@@ -33,9 +34,7 @@ sendgrid-django = "*"
3334
psycopg2-binary = "*"
3435
coreapi = "*"
3536
Django = "<3.0"
36-
numpy = "*"
3737
django-simple-history = "*"
38-
twisted = {version = "*",extras = ["tls"]}
3938
django-debug-toolbar = "*"
4039
google-api-python-client = "*"
4140
"oauth2client" = "*"
@@ -46,8 +45,8 @@ chargebee = "*"
4645
python-http-client = "<3.2.0" # 3.2.0 is the latest but throws an error on installation saying that it's not found
4746
django-health-check = "*"
4847
django-storages = "*"
49-
boto3 = "*"
5048
django-environ = "*"
5149
django-trench = "*"
5250
djoser = "*"
5351
influxdb-client = "*"
52+
django-ordered-model = "*"

Pipfile.lock

+191-382
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

readme.md

-1
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,6 @@ The application relies on the following environment variables to run:
120120
* `INFLUXDB_URL`: The URL for your InfluxDB database
121121
* `INFLUXDB_ORG`: The organisation string for your InfluxDB API call.
122122
* `GA_TABLE_ID`: GA table ID (view) to query when looking for organisation usage
123-
* `USE_S3_STORAGE`: 'True' to store static files in s3
124123
* `AWS_STORAGE_BUCKET_NAME`: bucket name to store static files. Required if `USE_S3_STORAGE' is true.
125124
* `AWS_S3_REGION_NAME`: region name of the static files bucket. Defaults to eu-west-2.
126125
* `ALLOWED_ADMIN_IP_ADDRESSES`: restrict access to the django admin console to a comma separated list of IP addresses (e.g. `127.0.0.1,127.0.0.2`)

src/api/serializers.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from rest_framework import serializers
2+
3+
4+
class ErrorSerializer(serializers.Serializer):
5+
message = serializers.CharField()

src/app/middleware.py

+5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
from django.conf import settings
22
from django.core.exceptions import PermissionDenied
33

4+
from util.logging import get_logger
5+
6+
logger = get_logger(__name__)
7+
48

59
class AdminWhitelistMiddleware:
610
def __init__(self, get_response):
@@ -12,6 +16,7 @@ def __call__(self, request):
1216
ip = x_forwarded_for.split(',')[0] if x_forwarded_for else request.META.get('REMOTE_ADDR')
1317
if settings.ALLOWED_ADMIN_IP_ADDRESSES and ip not in settings.ALLOWED_ADMIN_IP_ADDRESSES:
1418
# IP address not allowed!
19+
logger.info('Denying access to admin for ip address %s' % ip)
1520
raise PermissionDenied()
1621

1722
return self.get_response(request)

src/app/settings/common.py

+17-13
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
For the full list of settings and their values, see
1010
https://docs.djangoproject.com/en/1.9/ref/settings/
1111
"""
12-
import logging
1312
import os
1413
import warnings
1514
from importlib import reload
@@ -29,6 +28,8 @@
2928
PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
3029

3130
ENV = env('ENVIRONMENT', default='local')
31+
if ENV not in ('local', 'dev', 'staging', 'production'):
32+
warnings.warn('ENVIRONMENT env variable must be one of local, dev, staging or production')
3233

3334
if 'DJANGO_SECRET_KEY' not in os.environ:
3435
secret_key_gen()
@@ -106,6 +107,9 @@
106107
# health check plugins
107108
'health_check',
108109
'health_check.db',
110+
111+
# Used for ordering models (e.g. FeatureSegment)
112+
'ordered_model',
109113
]
110114

111115
if GOOGLE_ANALYTICS_KEY or INFLUXDB_TOKEN:
@@ -125,7 +129,10 @@
125129
),
126130
'PAGE_SIZE': 10,
127131
'UNICODE_JSON': False,
128-
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination'
132+
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
133+
'DEFAULT_THROTTLE_RATES': {
134+
'login': '1/s'
135+
}
129136
}
130137

131138
MIDDLEWARE = [
@@ -147,7 +154,9 @@
147154
if INFLUXDB_TOKEN:
148155
MIDDLEWARE.append('analytics.middleware.InfluxDBMiddleware')
149156

150-
if ENV != 'local':
157+
ALLOWED_ADMIN_IP_ADDRESSES = env.list('ALLOWED_ADMIN_IP_ADDRESSES', default=list())
158+
if len(ALLOWED_ADMIN_IP_ADDRESSES) > 0:
159+
warnings.warn('Restricting access to the admin site for ip addresses %s' % ', '.join(ALLOWED_ADMIN_IP_ADDRESSES))
151160
MIDDLEWARE.append('app.middleware.AdminWhitelistMiddleware')
152161

153162
ROOT_URLCONF = 'app.urls'
@@ -320,16 +329,6 @@
320329
}
321330
}
322331

323-
if env.bool('USE_S3_STORAGE', default=False):
324-
STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
325-
AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME']
326-
AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME', 'eu-west-2')
327-
AWS_LOCATION = 'static'
328-
AWS_DEFAULT_ACL = 'public-read'
329-
AWS_S3_ADDRESSING_STYLE = 'virtual'
330-
331-
ALLOWED_ADMIN_IP_ADDRESSES = env.list('ALLOWED_ADMIN_IP_ADDRESSES', default=list())
332-
333332
LOG_LEVEL = env.str('LOG_LEVEL', 'WARNING')
334333

335334
TRENCH_AUTH = {
@@ -368,3 +367,8 @@
368367
'user_list': ['custom_auth.permissions.CurrentUser'],
369368
}
370369
}
370+
371+
372+
# Github OAuth credentials
373+
GITHUB_CLIENT_ID = env.str('GITHUB_CLIENT_ID', '')
374+
GITHUB_CLIENT_SECRET = env.str('GITHUB_CLIENT_SECRET', '')

src/app/settings/master.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@
4040
REST_FRAMEWORK['PAGE_SIZE'] = 999
4141

4242
SECURE_SSL_REDIRECT = True
43-
SECURE_REDIRECT_EXEMPT = [r'^/$', r'^$'] # root is exempt as it's used for EB health checks
43+
SECURE_REDIRECT_EXEMPT = [r'^health$'] # /health is exempt as it's used for EB health checks

src/app/settings/staging.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@
4040
REST_FRAMEWORK['PAGE_SIZE'] = 999
4141

4242
SECURE_SSL_REDIRECT = True
43-
SECURE_REDIRECT_EXEMPT = [r'^/$', r'^$'] # root is exempt as it's used for EB health checks
43+
SECURE_REDIRECT_EXEMPT = [r'^health$'] # /health is exempt as it's used for EB health checks

src/app/urls.py

-5
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,5 @@
2222
if settings.DEBUG:
2323
import debug_toolbar
2424
urlpatterns = [
25-
# Django 2
26-
# path('__debug__/', include(debug_toolbar.urls)),
27-
28-
# For django versions before 2.0:
2925
url(r'^__debug__/', include(debug_toolbar.urls)),
30-
3126
] + urlpatterns

src/audit/models.py

+13-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
FEATURE_UPDATED_MESSAGE = "Flag / Remote Config updated: %s"
1010
SEGMENT_CREATED_MESSAGE = "New Segment created: %s"
1111
SEGMENT_UPDATED_MESSAGE = "Segment updated: %s"
12-
FEATURE_SEGMENT_UPDATED_MESSAGE = "Segment rules updated for flag: %s"
12+
FEATURE_SEGMENT_UPDATED_MESSAGE = "Segment rules updated for flag: %s in environment: %s"
1313
ENVIRONMENT_CREATED_MESSAGE = "New Environment created: %s"
1414
ENVIRONMENT_UPDATED_MESSAGE = "Environment updated: %s"
1515
FEATURE_STATE_UPDATED_MESSAGE = "Flag state / Remote Config value updated for feature: %s"
@@ -45,3 +45,15 @@ class Meta:
4545

4646
def __str__(self):
4747
return "Audit Log %s" % self.id
48+
49+
@classmethod
50+
def create_record(cls, obj, obj_type, log_message, author, project=None, environment=None):
51+
cls.objects.create(
52+
related_object_id=obj.id,
53+
related_object_type=obj_type.name,
54+
log=log_message,
55+
author=author,
56+
project=project,
57+
environment=environment
58+
)
59+

src/audit/signals.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55

66
from audit.models import AuditLog
77
from audit.serializers import AuditLogSerializer
8+
from util.logging import get_logger
89
from webhooks.webhooks import call_organisation_webhooks, WebhookEventType
910

10-
logger = logging.getLogger(__name__)
11-
logger.setLevel(logging.INFO)
11+
logger = get_logger(__name__)
1212

1313

1414
@receiver(post_save, sender=AuditLog)

src/custom_auth/oauth/exceptions.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
class GithubError(Exception):
2+
pass
3+
4+
5+
class GoogleError(Exception):
6+
pass
7+
8+
9+
class OAuthError(Exception):
10+
pass

src/custom_auth/oauth/github.py

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import requests
2+
from django.conf import settings
3+
from requests import RequestException
4+
5+
from custom_auth.oauth.exceptions import GithubError
6+
from custom_auth.oauth.helpers.github_helpers import convert_response_data_to_dictionary, get_first_and_last_name
7+
from util.logging import get_logger
8+
9+
GITHUB_API_URL = "https://api.github.com"
10+
GITHUB_OAUTH_URL = "https://github.com/login/oauth"
11+
12+
NON_200_ERROR_MESSAGE = "Github returned {} status code when getting an access token."
13+
14+
logger = get_logger(__name__)
15+
16+
17+
class GithubUser:
18+
def __init__(self, code: str, client_id: str = None, client_secret: str = None):
19+
self.client_id = client_id or settings.GITHUB_CLIENT_ID
20+
self.client_secret = client_secret or settings.GITHUB_CLIENT_SECRET
21+
22+
self.access_token = self._get_access_token(code)
23+
self.headers = {
24+
"Authorization": f"token {self.access_token}"
25+
}
26+
27+
def _get_access_token(self, code) -> str:
28+
data = {
29+
"code": code,
30+
"client_id": self.client_id,
31+
"client_secret": self.client_secret
32+
}
33+
response = requests.post(f"{GITHUB_OAUTH_URL}/access_token", data=data)
34+
35+
if response.status_code != 200:
36+
raise GithubError(NON_200_ERROR_MESSAGE.format(response.status_code))
37+
38+
response_json = convert_response_data_to_dictionary(response.text)
39+
if "error" in response_json:
40+
error_message = response_json["error_description"].replace("+", " ")
41+
raise GithubError(error_message)
42+
43+
return response_json["access_token"]
44+
45+
def get_user_info(self) -> dict:
46+
# TODO: use threads?
47+
try:
48+
return {
49+
**self._get_user_name_and_id(),
50+
"email": self._get_primary_email()
51+
}
52+
except RequestException:
53+
raise GithubError("Failed to communicate with the Github API.")
54+
55+
def _get_user_name_and_id(self):
56+
user_response = requests.get(f"{GITHUB_API_URL}/user", headers=self.headers)
57+
user_response_json = user_response.json()
58+
full_name = user_response_json.get("name")
59+
first_name, last_name = get_first_and_last_name(full_name) if full_name else ["", ""]
60+
return {
61+
"first_name": first_name,
62+
"last_name": last_name,
63+
"github_user_id": user_response_json.get("id")
64+
}
65+
66+
def _get_primary_email(self):
67+
emails_response = requests.get(f"{GITHUB_API_URL}/user/emails", headers=self.headers)
68+
69+
# response from github should be a list of dictionaries, this will find the first entry that is both verified
70+
# and marked as primary (there should only be one).
71+
primary_email_data = next(
72+
filter(lambda email_data: email_data["primary"] and email_data["verified"], emails_response.json()), None
73+
)
74+
75+
if not primary_email_data:
76+
raise GithubError("User does not have a verified email address with Github.")
77+
78+
return primary_email_data["email"]

src/custom_auth/oauth/google.py

+21-9
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,27 @@
11
import requests
2+
from requests import RequestException
3+
from rest_framework import status
4+
5+
from custom_auth.oauth.exceptions import GoogleError
26

37
USER_INFO_URL = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json&"
8+
NON_200_ERROR_MESSAGE = "Google returned {} status code when getting an access token."
49

510

611
def get_user_info(access_token):
7-
headers = {"Authorization": f"Bearer {access_token}"}
8-
response = requests.get(USER_INFO_URL, headers=headers)
9-
response_json = response.json()
10-
return {
11-
"email": response_json["email"],
12-
"first_name": response_json.get("given_name", ""),
13-
"last_name": response_json.get("family_name", ""),
14-
"google_user_id": response_json["id"]
15-
}
12+
try:
13+
headers = {"Authorization": f"Bearer {access_token}"}
14+
response = requests.get(USER_INFO_URL, headers=headers)
15+
16+
if response.status_code != status.HTTP_200_OK:
17+
raise GoogleError(NON_200_ERROR_MESSAGE.format(response.status_code))
18+
19+
response_json = response.json()
20+
return {
21+
"email": response_json["email"],
22+
"first_name": response_json.get("given_name", ""),
23+
"last_name": response_json.get("family_name", ""),
24+
"google_user_id": response_json["id"]
25+
}
26+
except RequestException:
27+
raise GoogleError("Failed to communicate with the Google API.")

src/custom_auth/oauth/helpers/__init__.py

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from custom_auth.oauth.exceptions import GithubError
2+
from util.logging import get_logger
3+
4+
logger = get_logger(__name__)
5+
6+
7+
def convert_response_data_to_dictionary(text: str) -> dict:
8+
try:
9+
response_data = {}
10+
for key, value in [param.split("=") for param in text.split("&")]:
11+
response_data[key] = value
12+
return response_data
13+
except ValueError:
14+
logger.warning("Malformed data received from Github (%s)" % text)
15+
raise GithubError("Malformed data received from Github")
16+
17+
18+
def get_first_and_last_name(full_name: str) -> list:
19+
if not full_name:
20+
return ["", ""]
21+
22+
names = full_name.strip().split(" ")
23+
return names if len(names) == 2 else [full_name, ""]

0 commit comments

Comments
 (0)