feat: Create split testing for multivariate

api/api/urls/v1.py

@@ -1,3 +1,7 @@
+from app_analytics.split_testing.views import (
+    ConversionEventViewSet,
+    SplitTestViewSet,
+)
 from app_analytics.views import SDKAnalyticsFlags, SelfHostedTelemetryAPIView
 from django.conf.urls import url
 from django.urls import include
@@ -27,6 +31,12 @@
 traits_router = routers.DefaultRouter()
 traits_router.register(r"", SDKTraits, basename="sdk-traits")
 
+split_testing_router = routers.DefaultRouter()
+split_testing_router.register(
+    r"conversion-events", ConversionEventViewSet, basename="conversion-events"
+)
+split_testing_router.register(r"", SplitTestViewSet, basename="split-tests")
+
 app_name = "v1"
 
 urlpatterns = [
@@ -47,8 +57,13 @@
     url(r"^flags/$", SDKFeatureStates.as_view(), name="flags"),
     url(r"^identities/$", SDKIdentities.as_view(), name="sdk-identities"),
     url(r"^traits/", include(traits_router.urls), name="traits"),
-    url(r"^analytics/flags/$", SDKAnalyticsFlags.as_view()),
-    url(r"^analytics/telemetry/$", SelfHostedTelemetryAPIView.as_view()),
+    url(r"^split-testing/", include(split_testing_router.urls), name="split-testing"),
+    url(r"^analytics/flags/$", SDKAnalyticsFlags.as_view(), name="analytics-flags"),
+    url(
+        r"^analytics/telemetry/$",
+        SelfHostedTelemetryAPIView.as_view(),
+        name="analytics-telemetry",
+    ),
     url(
         r"^environment-document/$",
         SDKEnvironmentAPIView.as_view(),

api/app/settings/common.py

@@ -164,6 +164,7 @@
     "softdelete",
     "metadata",
     "app_analytics",
+    "app_analytics.split_testing",
 ]
 
 SITE_ID = 1

api/app_analytics/migrations/0002_featureevaluationraw_identifier_and_index_feature.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.2.23 on 2024-01-02 16:35
+
+from django.db import migrations, models
+from core.migration_helpers import PostgresOnlyRunSQL
+
+
+class Migration(migrations.Migration):
+
+    atomic = False
+
+    dependencies = [
+        ('app_analytics', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='featureevaluationraw',
+            name='identifier',
+            field=models.CharField(default=None, max_length=2000, null=True),
+        ),
+        migrations.SeparateDatabaseAndState(
+            state_operations=[
+                migrations.AlterField(
+                    model_name='featureevaluationraw',
+                    name='feature_name',
+                    field=models.CharField(db_index=True, max_length=2000),
+                ),
+            ],
+            database_operations=[
+                PostgresOnlyRunSQL(
+                    'CREATE INDEX CONCURRENTLY "app_analytics_featureevaluationraw_feature_name_idx" ON "app_analytics_featureevaluationraw" ("feature_name");',
+                    reverse_sql='DROP INDEX CONCURRENTLY "app_analytics_featureevaluationraw_feature_name_idx";',
+                )
+            ],
+        ),
+    ]

api/app_analytics/models.py

@@ -73,11 +73,14 @@ def check_overlapping_buckets(self):
 
 
 class FeatureEvaluationRaw(models.Model):
-    feature_name = models.CharField(max_length=2000)
+    feature_name = models.CharField(db_index=True, max_length=2000)
     environment_id = models.PositiveIntegerField()
     evaluation_count = models.IntegerField(default=0)
     created_at = models.DateTimeField(auto_now_add=True)
 
+    # Identity identifier stored for tracking multivariate split testing.
+    identifier = models.CharField(max_length=2000, null=True, default=None)
+
 
 class FeatureEvaluationBucket(AbstractBucket):
     feature_name = models.CharField(max_length=2000)

api/app_analytics/split_testing/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AppAnalyticsConfig(AppConfig):
+    name = "app_analytics.split_testing"
+    label = "app_analytics_split_testing"

api/app_analytics/split_testing/helpers.py

@@ -0,0 +1,36 @@
+import numpy as np
+from scipy.stats import chi2_contingency
+
+
+def analyse_split_test(observed_matrix: np.array) -> tuple[float, float]:
+    # Replace zero values in order for the chi-squared results can
+    # be fully calculated. Don't worry about false results since
+    # the pvalue will be much too low to matter to the user.
+    replacement_value = 1
+    observed_matrix = np.where(observed_matrix == 0, replacement_value, observed_matrix)
+
+    # Calculate the results with correction set to `True` and the
+    # lambda set to what is commonly known as the G-Test.
+    results = chi2_contingency(
+        observed_matrix,
+        correction=True,
+        lambda_="log-likelihood",
+    )
+
+    # Return the most important result, the pvalue, as well as a
+    # possibly useful statistic addition for the frontend.
+    # Typically a pvalue of around 1% is ideal, though as large
+    # as 5% is acceptable for some tests.
+    return results.pvalue, results.statistic
+
+
+def gather_split_test_metrics(
+    evaluation_counts: dict[int, int], conversion_counts: dict[int, int]
+) -> tuple[float, float]:
+    _evaluation_counts = []
+    _conversion_counts = []
+    for mv_feature_option_id, evaluation_count in evaluation_counts.items():
+        _evaluation_counts.append(evaluation_count)
+        _conversion_counts.append(conversion_counts[mv_feature_option_id])
+    input_data = np.array([_conversion_counts, _evaluation_counts])
+    return analyse_split_test(input_data)

api/app_analytics/split_testing/migrations/0001_initial.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.23 on 2024-01-03 18:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('environments', '0033_add_environment_feature_state_version_logic'),
+        ('identities', '0002_alter_identity_index_together'),
+        ('features', '0062_alter_feature_segment_unique_together'),
+        ('multivariate', '0007_alter_boolean_values'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SplitTest',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('evaluation_count', models.PositiveIntegerField()),
+                ('conversion_count', models.PositiveIntegerField()),
+                ('pvalue', models.FloatField()),
+                ('statistic', models.FloatField()),
+                ('created_at', models.DateTimeField(auto_now_add=True, null=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('environment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='split_tests', to='environments.environment')),
+                ('feature', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='split_tests', to='features.feature')),
+                ('multivariate_feature_option', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='multivariate.multivariatefeatureoption')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ConversionEvent',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True, null=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('environment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='conversion_events', to='environments.environment')),
+                ('identity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='conversion_events', to='identities.identity')),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name='splittest',
+            constraint=models.UniqueConstraint(fields=('environment', 'feature', 'multivariate_feature_option'), name='unique_environment_feature_mvfo'),
+        ),
+    ]

api/app_analytics/split_testing/models.py

@@ -0,0 +1,55 @@
+from django.db import models
+
+from environments.identities.models import Identity
+from environments.models import Environment
+from features.models import Feature
+from features.multivariate.models import MultivariateFeatureOption
+
+
+class ConversionEvent(models.Model):
+    environment = models.ForeignKey(
+        Environment, related_name="conversion_events", on_delete=models.CASCADE
+    )
+    identity = models.ForeignKey(
+        Identity,
+        related_name="conversion_events",
+        on_delete=models.CASCADE,
+    )
+
+    created_at = models.DateTimeField(null=True, auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+
+class SplitTest(models.Model):
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["environment", "feature", "multivariate_feature_option"],
+                name="unique_environment_feature_mvfo",
+            )
+        ]
+
+    environment = models.ForeignKey(
+        Environment, related_name="split_tests", on_delete=models.CASCADE
+    )
+    feature = models.ForeignKey(
+        Feature, related_name="split_tests", on_delete=models.CASCADE
+    )
+    multivariate_feature_option = models.ForeignKey(
+        MultivariateFeatureOption, on_delete=models.CASCADE
+    )
+
+    # Populated from the existing split testing tasks.py to the
+    # number of unique identifiers for a single feature /
+    # environment combination. Multiple occurences ignored.
+    evaluation_count = models.PositiveIntegerField()
+    # from the ConversionEvent model for matching identifiers.
+    conversion_count = models.PositiveIntegerField()
+
+    # Split test metrics, where the pvalue is the most useful.
+    # See the analyse_split_test helpers function for more details.
+    pvalue = models.FloatField(null=False)
+    statistic = models.FloatField(null=False)
+
+    created_at = models.DateTimeField(null=True, auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)

api/app_analytics/split_testing/permissions.py

@@ -0,0 +1,23 @@
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.request import Request
+from rest_framework.viewsets import ModelViewSet
+
+from environments.models import Environment
+from environments.permissions.constants import VIEW_ENVIRONMENT
+
+
+class SplitTestPermissions(IsAuthenticated):
+    def has_permission(self, request: Request, view: ModelViewSet) -> bool:
+        if not super().has_permission(request, view):
+            return False
+
+        environment_id = request.query_params.get("environment_id")
+
+        if not environment_id:
+            return False
+
+        environment = Environment.objects.get(id=environment_id)
+
+        return request.user.has_environment_permission(
+            permission=VIEW_ENVIRONMENT, environment=environment
+        )

api/app_analytics/split_testing/serializers.py

@@ -0,0 +1,40 @@
+from rest_framework import serializers
+
+from environments.identities.models import Identity
+from features.multivariate.serializers import (
+    NestedMultivariateFeatureOptionSerializer,
+)
+from features.serializers import FeatureSerializer
+
+from .models import ConversionEvent, SplitTest
+
+
+class ConversionEventSerializer(serializers.Serializer):
+    identity_identifier = serializers.CharField(required=True)
+
+    def save(self, *args, **kwargs) -> ConversionEvent:
+        environment = self.context["request"].environment
+        identity = Identity.objects.get(
+            environment=environment,
+            identifier=self.validated_data["identity_identifier"],
+        )
+        return ConversionEvent.objects.create(
+            environment=environment,
+            identity=identity,
+        )
+
+
+class SplitTestSerializer(serializers.ModelSerializer):
+    feature = FeatureSerializer()
+    multivariate_feature_option = NestedMultivariateFeatureOptionSerializer()
+
+    class Meta:
+        model = SplitTest
+        fields = (
+            "feature",
+            "multivariate_feature_option",
+            "evaluation_count",
+            "conversion_count",
+            "pvalue",
+            "statistic",
+        )