coderd/experiments/experiments_test.go
Rama main · réplica de lectura del repositorio original.
package experiments_test
import (
"context"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/experiments"
"github.com/coder/coder/v2/coderd/experiments/experimentstest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
)
const (
scoped = codersdk.ExperimentExample
unscoped = codersdk.ExperimentWorkspaceUsage
sentinel = "sentinel-7f3a@example.test"
)
// countingStore records how often each Store method runs.
type countingStore struct {
experiments.Store
rules atomic.Int64
users atomic.Int64
}
func (s *countingStore) Rules(ctx context.Context) (map[codersdk.Experiment]experiments.StoredRule, error) {
s.rules.Add(1)
return s.Store.Rules(ctx)
}
func (s *countingStore) UserAttributes(ctx context.Context, userID uuid.UUID) (experiments.User, error) {
s.users.Add(1)
return s.Store.UserAttributes(ctx, userID)
}
var testUser = experiments.User{
ID: "6d8e1a52-6f2c-4a4e-9b3a-0d1d8f1f7f10",
Username: "alice",
Email: "Alice@Example.com",
Roles: []string{"owner"},
Organizations: []string{"coder", "acme"},
Groups: []string{"coder/Everyone", "coder/beta", "acme/Everyone"},
}
func storeWith(t *testing.T, userID uuid.UUID, rule *experiments.Rule) *countingStore {
t.Helper()
s := experimentstest.Store{Users: map[uuid.UUID]experiments.User{userID: testUser}}
if rule != nil {
s.StoredRules = map[codersdk.Experiment]experiments.StoredRule{
scoped: experimentstest.StoredRule(t, *rule),
}
}
return &countingStore{Store: s}
}
func rawStore(userID uuid.UUID, raw string) *countingStore {
return &countingStore{Store: experimentstest.Store{
Users: map[uuid.UUID]experiments.User{userID: testUser},
StoredRules: map[codersdk.Experiment]experiments.StoredRule{scoped: {Value: []byte(raw)}},
}}
}
func newEvaluator(t *testing.T, store experiments.Store, static codersdk.Experiments) (*experiments.Evaluator, *testutil.FakeSink) {
t.Helper()
sink := testutil.NewFakeSink(t)
e, err := experiments.New(sink.Logger(), store, static)
require.NoError(t, err)
return e, sink
}
func condition(src string) *experiments.Rule {
return &experiments.Rule{Mode: experiments.ModeCondition, Condition: src, Revision: 3}
}
// requireLogsExclude fails if any captured entry contains one of the
// forbidden strings in its message or fields.
func requireLogsExclude(t *testing.T, sink *testutil.FakeSink, forbidden ...string) {
t.Helper()
for _, entry := range sink.Entries() {
text := entry.Message + " " + fmt.Sprint(entry.Fields)
for _, f := range forbidden {
require.NotContains(t, text, f)
}
}
}
// requireLogField fails unless some captured entry has field name set to
// want.
func requireLogField(t *testing.T, sink *testutil.FakeSink, name string, want any) {
t.Helper()
for _, entry := range sink.Entries() {
for _, field := range entry.Fields {
if field.Name == name && fmt.Sprint(field.Value) == fmt.Sprint(want) {
return
}
}
}
require.Failf(t, "missing log field", "%s=%v", name, want)
}
func TestNew(t *testing.T) {
t.Parallel()
_, err := experiments.New(slog.Make(), nil, nil)
require.Error(t, err)
}
func TestValidateRule(t *testing.T) {
t.Parallel()
valid := []experiments.Rule{
{Mode: experiments.ModeInherit},
{Mode: experiments.ModeOn},
{Mode: experiments.ModeOff},
{Mode: experiments.ModeCondition, Condition: `user.email == "a@b.c" || "admin" in user.roles`},
{Mode: experiments.ModeCondition, Condition: strings.Repeat(" ", experiments.MaxConditionSize-4) + "true"},
}
for _, rule := range valid {
require.NoError(t, experiments.ValidateRule(scoped, rule), "mode %q", rule.Mode)
}
invalid := map[string]experiments.Rule{
"syntax": {Mode: experiments.ModeCondition, Condition: `user.email ==`},
"non-bool": {Mode: experiments.ModeCondition, Condition: `user.email`},
"unknown field": {Mode: experiments.ModeCondition, Condition: `user.nickname == "x"`},
"unknown var": {Mode: experiments.ModeCondition, Condition: `request.ip == "x"`},
"size": {Mode: experiments.ModeCondition, Condition: strings.Repeat(" ", experiments.MaxConditionSize-3) + "true"},
"empty": {Mode: experiments.ModeCondition},
"unknown mode": {Mode: "sometimes"},
"stray condition": {Mode: experiments.ModeOn, Condition: "true"},
}
for name, rule := range invalid {
require.Error(t, experiments.ValidateRule(scoped, rule), name)
}
require.Error(t, experiments.ValidateRule(unscoped, experiments.Rule{Mode: experiments.ModeOn}))
}
func TestDiagnosticsNotLogged(t *testing.T) {
t.Parallel()
cases := map[string]struct {
src string
compileFail bool
// message is part of the CEL message text for this failure.
message string
category string
}{
"syntax": {src: `user.email == "` + sentinel + `" &&`, compileFail: true, message: "Syntax error", category: "syntax"},
"type": {src: `user.email == 1 || user.email == "` + sentinel + `"`, compileFail: true, message: "no matching overload", category: "type"},
"runtime": {src: `user.email == "x" || int("` + sentinel + `") == 1`, message: "conversion error", category: "eval"},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
userID := uuid.New()
validateErr := experiments.ValidateRule(scoped, *condition(tc.src))
if tc.compileFail {
require.Error(t, validateErr)
} else {
require.NoError(t, validateErr)
}
e, sink := newEvaluator(t, storeWith(t, userID, condition(tc.src)), codersdk.Experiments{scoped})
require.False(t, e.Enabled(ctx, userID, scoped))
require.NotEmpty(t, sink.Entries(), "failure must be logged")
forbidden := []string{sentinel, tc.src, tc.message}
if validateErr != nil {
require.Contains(t, validateErr.Error(), tc.message)
forbidden = append(forbidden, validateErr.Error())
}
requireLogsExclude(t, sink, forbidden...)
requireLogField(t, sink, "category", tc.category)
requireLogField(t, sink, "experiment", scoped)
requireLogField(t, sink, "revision", int64(3))
})
}
}
func TestDecisions(t *testing.T) {
t.Parallel()
cases := map[string]struct {
rule *experiments.Rule
static bool
want bool
wantUserLoads int64
}{
"none static on": {static: true, want: true},
"none static off": {static: false, want: false},
"inherit static on": {rule: &experiments.Rule{Mode: experiments.ModeInherit}, static: true, want: true},
"inherit static off": {rule: &experiments.Rule{Mode: experiments.ModeInherit}, static: false, want: false},
"on": {rule: &experiments.Rule{Mode: experiments.ModeOn}, want: true},
"off static on": {rule: &experiments.Rule{Mode: experiments.ModeOff}, static: true, want: false},
"email": {rule: condition(`user.email == "Alice@Example.com"`), want: true, wantUserLoads: 1},
"email case": {rule: condition(`user.email == "alice@example.com"`), static: true, want: false, wantUserLoads: 1},
"username id": {rule: condition(`user.username == "alice" && user.id == "` + testUser.ID + `"`), want: true, wantUserLoads: 1},
"role": {rule: condition(`"owner" in user.roles`), want: true, wantUserLoads: 1},
"missing role": {rule: condition(`"member" in user.roles`), want: false, wantUserLoads: 1},
"org": {rule: condition(`"acme" in user.organizations`), want: true, wantUserLoads: 1},
"group": {rule: condition(`"coder/beta" in user.groups`), want: true, wantUserLoads: 1},
"everyone": {rule: condition(`"acme/Everyone" in user.groups`), want: true, wantUserLoads: 1},
"and": {rule: condition(`"coder" in user.organizations && "acme/beta" in user.groups`), want: false, wantUserLoads: 1},
"or": {rule: condition(`"other" in user.organizations || "coder/beta" in user.groups`), want: true, wantUserLoads: 1},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
userID := uuid.New()
store := storeWith(t, userID, tc.rule)
var static codersdk.Experiments
if tc.static {
static = codersdk.Experiments{scoped}
}
e, _ := newEvaluator(t, store, static)
require.Equal(t, tc.want, e.Enabled(ctx, userID, scoped))
require.EqualValues(t, 1, store.rules.Load())
require.Equal(t, tc.wantUserLoads, store.users.Load())
})
}
}
func TestUnscopedUsesStatic(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
// Rules for experiments outside ExperimentsUserScoped are inert, and
// the subject does not matter.
store := &countingStore{Store: experimentstest.Store{
RulesErr: xerrors.New("boom"),
}}
e, _ := newEvaluator(t, store, codersdk.Experiments{unscoped})
require.True(t, e.Enabled(ctx, uuid.Nil, unscoped))
require.EqualValues(t, 0, store.rules.Load())
store = &countingStore{Store: experimentstest.Store{StoredRules: map[codersdk.Experiment]experiments.StoredRule{
unscoped: experimentstest.StoredRule(t, experiments.Rule{Mode: experiments.ModeOff}),
}}}
e, _ = newEvaluator(t, store, codersdk.Experiments{unscoped})
require.True(t, e.Enabled(ctx, uuid.New(), unscoped))
}
func TestFailClosed(t *testing.T) {
t.Parallel()
// manyGroups makes nested comprehensions exceed the cost limit.
manyGroups := make([]string, 200)
for i := range manyGroups {
manyGroups[i] = fmt.Sprintf("coder/g%d", i)
}
cases := map[string]struct {
store func(t *testing.T, userID uuid.UUID) experiments.Store
ctx func(ctx context.Context) context.Context
nilSubject bool
wantCategory string
}{
"rules read error": {store: func(_ *testing.T, _ uuid.UUID) experiments.Store {
return experimentstest.Store{RulesErr: xerrors.New("db down")}
}},
"malformed json": {store: func(_ *testing.T, userID uuid.UUID) experiments.Store {
return rawStore(userID, `{"mode": "inherit"`)
}},
"unknown mode": {store: func(_ *testing.T, userID uuid.UUID) experiments.Store {
return rawStore(userID, `{"mode": "percent", "revision": 2}`)
}},
"condition without text": {store: func(_ *testing.T, userID uuid.UUID) experiments.Store {
return rawStore(userID, `{"mode": "condition", "revision": 2}`)
}},
"attribute load error": {store: func(t *testing.T, _ uuid.UUID) experiments.Store {
return experimentstest.Store{
StoredRules: map[codersdk.Experiment]experiments.StoredRule{
scoped: experimentstest.StoredRule(t, *condition(`true`)),
},
UserErr: xerrors.New("db down"),
}
}},
"compile error": {store: func(t *testing.T, userID uuid.UUID) experiments.Store {
return storeWith(t, userID, condition(`user.nickname == "x"`))
}},
"runtime error": {store: func(t *testing.T, userID uuid.UUID) experiments.Store {
return storeWith(t, userID, condition(`user.roles[5] == "owner"`))
}},
"cost overrun": {store: func(t *testing.T, userID uuid.UUID) experiments.Store {
return experimentstest.Store{
StoredRules: map[codersdk.Experiment]experiments.StoredRule{
scoped: experimentstest.StoredRule(t, *condition(`user.groups.all(a, user.groups.all(b, a != b || true))`)),
},
Users: map[uuid.UUID]experiments.User{userID: {Groups: manyGroups}},
}
}},
"canceled context": {
store: func(t *testing.T, userID uuid.UUID) experiments.Store {
return storeWith(t, userID, condition(`true`))
},
ctx: canceledContext,
wantCategory: "canceled",
},
// The store ignores cancellation, so the rules read succeeds and
// only the decision itself can notice the canceled context.
"canceled context with on rule": {
store: func(t *testing.T, userID uuid.UUID) experiments.Store {
return storeWith(t, userID, &experiments.Rule{Mode: experiments.ModeOn, Revision: 2})
},
ctx: canceledContext,
wantCategory: "canceled",
},
"canceled context without rule and static on": {
store: func(t *testing.T, userID uuid.UUID) experiments.Store {
return storeWith(t, userID, nil)
},
ctx: canceledContext,
wantCategory: "canceled",
},
"nil subject": {
store: func(t *testing.T, userID uuid.UUID) experiments.Store {
return storeWith(t, userID, &experiments.Rule{Mode: experiments.ModeInherit})
},
nilSubject: true,
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
if tc.ctx != nil {
ctx = tc.ctx(ctx)
}
userID := uuid.New()
store := tc.store(t, userID)
if tc.nilSubject {
userID = uuid.Nil
}
e, sink := newEvaluator(t, store, codersdk.Experiments{scoped, unscoped})
require.False(t, e.Enabled(ctx, userID, scoped))
require.Equal(t, codersdk.Experiments{unscoped}, e.EnabledExperiments(ctx, userID))
require.NotEmpty(t, sink.Entries(), "fail-closed decisions must be logged")
if tc.wantCategory != "" {
requireLogField(t, sink, "category", tc.wantCategory)
}
})
}
}
// TestRuleWithoutPositiveRevisionIsMalformed covers values written outside
// WriteRule: every stored rule has revision 1 or higher, so a value without
// one decides off even when its mode is valid.
func TestRuleWithoutPositiveRevisionIsMalformed(t *testing.T) {
t.Parallel()
src := fmt.Sprintf(`user.email != %q`, sentinel)
cases := map[string]string{
"on revision 0": `{"mode":"on","revision":0}`,
"on negative revision": `{"mode":"on","revision":-2}`,
"on missing revision": `{"mode":"on"}`,
"inherit missing": `{"mode":"inherit"}`,
"condition revision 0": fmt.Sprintf(`{"mode":"condition","condition":%q,"revision":0}`, src),
"condition missing": fmt.Sprintf(`{"mode":"condition","condition":%q}`, src),
"condition negative": fmt.Sprintf(`{"mode":"condition","condition":%q,"revision":-1}`, src),
"undecodable revision 0": `{"mode":"on","revision":0,"updated_by":"` + sentinel + `"}`,
}
for name, raw := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
userID := uuid.New()
e, sink := newEvaluator(t, rawStore(userID, raw), codersdk.Experiments{scoped, unscoped})
require.False(t, e.Enabled(ctx, userID, scoped))
require.Equal(t, codersdk.Experiments{unscoped}, e.EnabledExperiments(ctx, userID))
requireLogField(t, sink, "category", "malformed")
requireLogField(t, sink, "experiment", scoped)
requireLogsExclude(t, sink, sentinel, raw)
})
}
}
// TestCanceledStoreReadIsNotAnError checks that a store read failing
// because the caller went away logs at Debug with category canceled, while
// other read failures stay errors. Every case decides off.
func TestCanceledStoreReadIsNotAnError(t *testing.T) {
t.Parallel()
canceled := xerrors.Errorf("read: %w", context.Canceled)
expired := xerrors.Errorf("read: %w", context.DeadlineExceeded)
cases := map[string]struct {
store func(t *testing.T) experiments.Store
wantCategory string
wantLevel slog.Level
}{
"rules canceled": {
store: func(*testing.T) experiments.Store { return experimentstest.Store{RulesErr: canceled} },
wantCategory: "canceled", wantLevel: slog.LevelDebug,
},
"rules deadline": {
store: func(*testing.T) experiments.Store { return experimentstest.Store{RulesErr: expired} },
wantCategory: "canceled", wantLevel: slog.LevelDebug,
},
"rules failure": {
store: func(*testing.T) experiments.Store { return experimentstest.Store{RulesErr: xerrors.New("db down")} },
wantCategory: "read", wantLevel: slog.LevelError,
},
"attributes canceled": {
store: func(t *testing.T) experiments.Store {
return experimentstest.Store{
StoredRules: map[codersdk.Experiment]experiments.StoredRule{scoped: experimentstest.StoredRule(t, *condition(`true`))},
UserErr: canceled,
}
},
wantCategory: "canceled", wantLevel: slog.LevelDebug,
},
"attributes failure": {
store: func(t *testing.T) experiments.Store {
return experimentstest.Store{
StoredRules: map[codersdk.Experiment]experiments.StoredRule{scoped: experimentstest.StoredRule(t, *condition(`true`))},
UserErr: xerrors.New("db down"),
}
},
wantCategory: "read", wantLevel: slog.LevelError,
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
e, sink := newEvaluator(t, tc.store(t), codersdk.Experiments{scoped})
require.False(t, e.Enabled(ctx, uuid.New(), scoped))
entries := sink.Entries()
require.Len(t, entries, 1)
require.Equal(t, tc.wantLevel, entries[0].Level)
requireLogField(t, sink, "category", tc.wantCategory)
})
}
}
func canceledContext(ctx context.Context) context.Context {
ctx, cancel := context.WithCancel(ctx)
cancel()
return ctx
}
// TestRuleChangesApplyToNextCall edits the stored rule between calls on one
// Evaluator: each call must reflect the current rule, never an earlier
// program or decision, including after a failed read.
func TestRuleChangesApplyToNextCall(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
userID := uuid.New()
store := &swapStore{Store: storeWith(t, userID, nil)}
e, _ := newEvaluator(t, store, nil)
steps := []struct {
name string
// rule nil means the rules read fails.
rule *experiments.Rule
want bool
}{
{"matching condition", condition(`"owner" in user.roles`), true},
{"edited condition", condition(`"member" in user.roles`), false},
{"original condition again", condition(`"owner" in user.roles`), true},
{"read failure", nil, false},
{"after read failure", condition(`"owner" in user.roles`), true},
{"off", &experiments.Rule{Mode: experiments.ModeOff}, false},
{"on", &experiments.Rule{Mode: experiments.ModeOn}, true},
}
for _, step := range steps {
if step.rule == nil {
store.rules.Store(nil)
} else {
rules := map[codersdk.Experiment]experiments.StoredRule{
scoped: experimentstest.StoredRule(t, *step.rule),
}
store.rules.Store(&rules)
}
require.Equal(t, step.want, e.Enabled(ctx, userID, scoped), step.name)
}
}
// swapStore returns the rules currently stored in rules, and fails the read
// while rules is nil.
type swapStore struct {
experiments.Store
rules atomic.Pointer[map[codersdk.Experiment]experiments.StoredRule]
}
func (s *swapStore) Rules(context.Context) (map[codersdk.Experiment]experiments.StoredRule, error) {
rules := s.rules.Load()
if rules == nil {
return nil, xerrors.New("db down")
}
return *rules, nil
}
func TestEnabledExperiments(t *testing.T) {
t.Parallel()
unknown := codersdk.Experiment("not-a-real-experiment")
t.Run("KeepsStaticOrderAndUnknown", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
userID := uuid.New()
static := codersdk.Experiments{unknown, scoped, unscoped}
store := storeWith(t, userID, nil)
e, _ := newEvaluator(t, store, static)
require.Equal(t, static, e.EnabledExperiments(ctx, userID))
require.EqualValues(t, 1, store.rules.Load())
require.EqualValues(t, 0, store.users.Load())
})
t.Run("RemovesDecidedOff", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
userID := uuid.New()
store := storeWith(t, userID, condition(`"nobody" in user.roles`))
e, _ := newEvaluator(t, store, codersdk.Experiments{unknown, scoped, unscoped})
require.Equal(t, codersdk.Experiments{unknown, unscoped}, e.EnabledExperiments(ctx, userID))
require.EqualValues(t, 1, store.rules.Load())
require.EqualValues(t, 1, store.users.Load())
})
t.Run("AppendsDecidedOn", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
userID := uuid.New()
store := storeWith(t, userID, condition(`"owner" in user.roles`))
e, _ := newEvaluator(t, store, codersdk.Experiments{unscoped, unknown})
require.Equal(t, codersdk.Experiments{unscoped, unknown, scoped}, e.EnabledExperiments(ctx, userID))
require.EqualValues(t, 1, store.rules.Load())
require.EqualValues(t, 1, store.users.Load())
})
}
func TestConcurrentEvaluation(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
userID := uuid.New()
sources := []string{`"owner" in user.roles`, `"member" in user.roles`, `user.username == "alice"`}
wants := []bool{true, false, true}
rules := make([]map[codersdk.Experiment]experiments.StoredRule, len(sources))
for i, src := range sources {
rules[i] = map[codersdk.Experiment]experiments.StoredRule{
scoped: experimentstest.StoredRule(t, *condition(src)),
}
}
// Each goroutine selects its rule through the context, so all of them
// share one evaluator and its program cache.
store := &ctxStore{Store: storeWith(t, userID, nil), rules: rules}
e, _ := newEvaluator(t, store, nil)
var wg sync.WaitGroup
for g := range 12 {
wg.Add(1)
go func() {
defer wg.Done()
i := g % len(sources)
gctx := context.WithValue(ctx, ruleIndexKey{}, i)
for range 50 {
assert.Equal(t, wants[i], e.Enabled(gctx, userID, scoped), sources[i])
}
}()
}
wg.Wait()
}
type ruleIndexKey struct{}
// ctxStore returns the rules selected by the ruleIndexKey context value.
type ctxStore struct {
experiments.Store
rules []map[codersdk.Experiment]experiments.StoredRule
}
func (s *ctxStore) Rules(ctx context.Context) (map[codersdk.Experiment]experiments.StoredRule, error) {
i, ok := ctx.Value(ruleIndexKey{}).(int)
if !ok {
return nil, xerrors.New("missing rule index")
}
return s.rules[i], nil
}