From 727ba3fbf98aa4505abb803a306fb1706f5ee682 Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Mon, 24 Feb 2020 15:05:09 -0500
Subject: [PATCH 01/11] Adds redis sentinel support

Fixes #1553
---
 .drone.yml                            | 11 ++++++
 cmd/proxy/actions/app_proxy.go        | 10 ++++++
 config.dev.toml                       | 11 ++++++
 docker-compose.yml                    |  8 +++++
 pkg/config/singleflight.go            | 13 ++++++--
 pkg/stash/with_redis_sentinel.go      | 25 ++++++++++++++
 pkg/stash/with_redis_sentinel_test.go | 48 +++++++++++++++++++++++++++
 7 files changed, 124 insertions(+), 2 deletions(-)
 create mode 100644 pkg/stash/with_redis_sentinel.go
 create mode 100644 pkg/stash/with_redis_sentinel_test.go

diff --git a/.drone.yml b/.drone.yml
index 9beb1aaf3..7d250f997 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -46,6 +46,9 @@ steps:
     ATHENS_MONGO_STORAGE_URL: mongodb://mongo:27017
     ATHENS_MINIO_ENDPOINT: minio:9000
     REDIS_TEST_ENDPOINT: redis:6379
+    REDIS_SENTINEL_TEST_ENDPOINT: redis-sentinel:23679
+    REDIS_SENTINEL_TEST_MASTER_NAME: redis-1
+    REDIS_SENTINEL_TEST_PASSWORD: sekret
     GCS_SERVICE_ACCOUNT:
       from_secret: GCS_SERVICE_ACCOUNT
     GCS_PROJECT_ID: 
@@ -154,6 +157,14 @@ services:
   image: redis
   ports:
     - 6379
+- name: redis-sentinel
+  image: bitnami/redis-sentinel
+  environment:
+    REDIS_MASTER_SET: redis-1
+    REDIS_SENTINEL_PASSWORD: sekret
+    REDIS_SENTINEL_QUORUM: "1"
+  ports:
+    - 26379
 - name: athens-proxy
   image: gomods/athens:canary
   pull: always
diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go
index 1a7058375..f5b2b54c4 100644
--- a/cmd/proxy/actions/app_proxy.go
+++ b/cmd/proxy/actions/app_proxy.go
@@ -133,6 +133,16 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper,
 			return nil, fmt.Errorf("Redis config must be present")
 		}
 		return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, checker)
+	case "redis-sentinel":
+		if c.SingleFlight == nil || c.SingleFlight.Redis == nil {
+			return nil, fmt.Errorf("Redis config must be present")
+		}
+		return stash.WithRedisSentinelLock(
+			c.SingleFlight.RedisSentinel.Endpoints,
+			c.SingleFlight.RedisSentinel.MasterName,
+			c.SingleFlight.RedisSentinel.SentinelPassword,
+			checker,
+		)
 	case "gcp":
 		if c.StorageType != "gcp" {
 			return nil, fmt.Errorf("gcp SingleFlight only works with a gcp storage type and not: %v", c.StorageType)
diff --git a/config.dev.toml b/config.dev.toml
index cdb1f29c7..db52697ea 100755
--- a/config.dev.toml
+++ b/config.dev.toml
@@ -290,6 +290,17 @@ SingleFlightType = "memory"
         # TODO(marwan): enable multiple endpoints for redis clusters.
         # Env override: ATHENS_REDIS_ENDPOINT
         Endpoint = "127.0.0.1:6379"
+    [SingleFlight.RedisSentinel]
+        # Endpoints is the redis sentinel endpoints to discover a redis 
+        # master for a SingleFlight lock.
+        # Env override: ATHENS_REDIS_SENTINEL_ENDPOINTS
+        Endpoint = ["127.0.0.1:26379"]
+        # MasterName is the redis sentinel master name to use to discover
+        # the master for a SingleFlight lock
+        MasterName = "redis-1"
+        # SentinelPassword is an optional password for authenticating with
+        # redis sentinel
+        SentinelPassword = "sekret"
 
 [Storage]
     # Only storage backends that are specified in Proxy.StorageType are required here
diff --git a/docker-compose.yml b/docker-compose.yml
index 8b04efcb0..80a2fc670 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -81,6 +81,14 @@ services:
     image: redis
     ports: 
       - 6379:6379
+  redis-sentinel:
+    image: bitnami/redis-sentinel
+    environment:
+      - REDIS_MASTER_SET=redis-1
+      - REDIS_SENTINEL_PASSWORD=sekret
+      - REDIS_SENTINEL_QUORUM=1
+    ports:
+      - 23679:23679
   etcd0:
     image: quay.io/coreos/etcd
     ports:
diff --git a/pkg/config/singleflight.go b/pkg/config/singleflight.go
index 361f4bc50..7c0f9f86b 100644
--- a/pkg/config/singleflight.go
+++ b/pkg/config/singleflight.go
@@ -4,8 +4,9 @@ package config
 // backend configurations for a distributed
 // lock or single flight mechanism.
 type SingleFlight struct {
-	Etcd  *Etcd
-	Redis *Redis
+	Etcd          *Etcd
+	Redis         *Redis
+	RedisSentinel *RedisSentinel
 }
 
 // Etcd holds client side configuration
@@ -20,3 +21,11 @@ type Etcd struct {
 type Redis struct {
 	Endpoint string `envconfig:"ATHENS_REDIS_ENDPOINT"`
 }
+
+// RedisSentinel is the configuration for using redis with sentinel
+// for SingleFlight
+type RedisSentinel struct {
+	Endpoints        []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"`
+	MasterName       string   `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"`
+	SentinelPassword string   `envconfig:"ATHENS_REDIS_SENTINEL_PASSWORD"`
+}
diff --git a/pkg/stash/with_redis_sentinel.go b/pkg/stash/with_redis_sentinel.go
new file mode 100644
index 000000000..ba23bb8a7
--- /dev/null
+++ b/pkg/stash/with_redis_sentinel.go
@@ -0,0 +1,25 @@
+package stash
+
+import (
+	"github.com/go-redis/redis"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/storage"
+)
+
+// WithRedisSentinelLock returns a distributed singleflight
+// with a redis cluster that utilizes sentinel for quorum and failover
+func WithRedisSentinelLock(endpoints []string, master, password string, checker storage.Checker) (Wrapper, error) {
+	const op errors.Op = "stash.WithRedisSentinelLock"
+	client := redis.NewFailoverClient(&redis.FailoverOptions{
+		MasterName:    master,
+		SentinelAddrs: endpoints,
+		Password:      password,
+	})
+	_, err := client.Ping().Result()
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	return func(s Stasher) Stasher {
+		return &redisLock{client, s, checker}
+	}, nil
+}
diff --git a/pkg/stash/with_redis_sentinel_test.go b/pkg/stash/with_redis_sentinel_test.go
new file mode 100644
index 000000000..5b3c05266
--- /dev/null
+++ b/pkg/stash/with_redis_sentinel_test.go
@@ -0,0 +1,48 @@
+package stash
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/gomods/athens/pkg/storage/mem"
+	"golang.org/x/sync/errgroup"
+)
+
+// WithRedisLock will ensure that 5 concurrent requests will all get the first request's
+// response. We can ensure that because only the first response does not return an error
+// and therefore all 5 responses should have no error.
+func TestWithRedisSentinelLock(t *testing.T) {
+	endpoint := os.Getenv("REDIS_SENTINEL_TEST_ENDPOINT")
+	masterName := os.Getenv("REDIS_SENTINEL_TEST_MASTER_NAME")
+	password := os.Getenv("REDIS_SENTINEL_TEST_PASSWORD")
+	if len(endpoint) == 0 || len(masterName) == 0 || len(password) == 0 {
+		t.SkipNow()
+	}
+	strg, err := mem.NewStorage()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ms := &mockRedisStasher{strg: strg}
+	wrapper, err := WithRedisSentinelLock([]string{endpoint}, masterName, password, strg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := wrapper(ms)
+
+	var eg errgroup.Group
+	for i := 0; i < 5; i++ {
+		eg.Go(func() error {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+			defer cancel()
+			_, err := s.Stash(ctx, "mod", "ver")
+			return err
+		})
+	}
+
+	err = eg.Wait()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

From 815ad26e685e8304fb051a0ca6742db45a7cf697 Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Mon, 24 Feb 2020 15:15:33 -0500
Subject: [PATCH 02/11] Fix redis-sentinel test hostnames

---
 .drone.yml         | 1 +
 docker-compose.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.drone.yml b/.drone.yml
index 7d250f997..740d366f4 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -160,6 +160,7 @@ services:
 - name: redis-sentinel
   image: bitnami/redis-sentinel
   environment:
+    REDIS_MASTER_HOST: redis
     REDIS_MASTER_SET: redis-1
     REDIS_SENTINEL_PASSWORD: sekret
     REDIS_SENTINEL_QUORUM: "1"
diff --git a/docker-compose.yml b/docker-compose.yml
index 80a2fc670..15a5e8592 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -84,6 +84,7 @@ services:
   redis-sentinel:
     image: bitnami/redis-sentinel
     environment:
+      - REDIS_MASTER_HOST=redis
       - REDIS_MASTER_SET=redis-1
       - REDIS_SENTINEL_PASSWORD=sekret
       - REDIS_SENTINEL_QUORUM=1

From e5a4f770c9ad2ca90ba47f1c5ad8ebe10cf2e15e Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Mon, 24 Feb 2020 15:29:08 -0500
Subject: [PATCH 03/11] Fix redis master name again

---
 .drone.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.drone.yml b/.drone.yml
index 740d366f4..a6e6f84f1 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -47,7 +47,7 @@ steps:
     ATHENS_MINIO_ENDPOINT: minio:9000
     REDIS_TEST_ENDPOINT: redis:6379
     REDIS_SENTINEL_TEST_ENDPOINT: redis-sentinel:23679
-    REDIS_SENTINEL_TEST_MASTER_NAME: redis-1
+    REDIS_SENTINEL_TEST_MASTER_NAME: redis
     REDIS_SENTINEL_TEST_PASSWORD: sekret
     GCS_SERVICE_ACCOUNT:
       from_secret: GCS_SERVICE_ACCOUNT

From b65258a87f1014a4d0bead4038992e64380a11a5 Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Mon, 24 Feb 2020 15:46:24 -0500
Subject: [PATCH 04/11] Fix redis sentinel port in tests

---
 .drone.yml         | 4 ++--
 docker-compose.yml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index a6e6f84f1..016366c86 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -46,8 +46,8 @@ steps:
     ATHENS_MONGO_STORAGE_URL: mongodb://mongo:27017
     ATHENS_MINIO_ENDPOINT: minio:9000
     REDIS_TEST_ENDPOINT: redis:6379
-    REDIS_SENTINEL_TEST_ENDPOINT: redis-sentinel:23679
-    REDIS_SENTINEL_TEST_MASTER_NAME: redis
+    REDIS_SENTINEL_TEST_ENDPOINT: redis-sentinel:26379
+    REDIS_SENTINEL_TEST_MASTER_NAME: redis-1
     REDIS_SENTINEL_TEST_PASSWORD: sekret
     GCS_SERVICE_ACCOUNT:
       from_secret: GCS_SERVICE_ACCOUNT
diff --git a/docker-compose.yml b/docker-compose.yml
index 15a5e8592..18d84d4f3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -89,7 +89,7 @@ services:
       - REDIS_SENTINEL_PASSWORD=sekret
       - REDIS_SENTINEL_QUORUM=1
     ports:
-      - 23679:23679
+      - 26379:26379
   etcd0:
     image: quay.io/coreos/etcd
     ports:

From 8791b2e9388db544c3ae642a8d028da4c43eee93 Mon Sep 17 00:00:00 2001
From: Ted Wexler <ted@stuckinacan.com>
Date: Mon, 24 Feb 2020 21:19:56 -0500
Subject: [PATCH 05/11] Upgrade the redis client

---
 config.dev.toml                  |  4 ++--
 go.mod                           |  2 ++
 go.sum                           | 21 +++++++++++++++++++++
 pkg/stash/with_redis.go          | 14 ++++++--------
 pkg/stash/with_redis_sentinel.go | 20 ++++++++++++++++----
 5 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/config.dev.toml b/config.dev.toml
index db52697ea..0ef9b125c 100755
--- a/config.dev.toml
+++ b/config.dev.toml
@@ -276,7 +276,7 @@ DownloadURL = ""
 # that only one module is ever written even when concurrent saves happen
 # at the same time.
 # Env override: ATHENS_SINGLE_FLIGHT_TYPE
-SingleFlightType = "memory"
+SingleFlightType = "redis-sentinel"
 
 [SingleFlight]
     [SingleFlight.Etcd]
@@ -294,7 +294,7 @@ SingleFlightType = "memory"
         # Endpoints is the redis sentinel endpoints to discover a redis 
         # master for a SingleFlight lock.
         # Env override: ATHENS_REDIS_SENTINEL_ENDPOINTS
-        Endpoint = ["127.0.0.1:26379"]
+        Endpoints = ["127.0.0.1:26379"]
         # MasterName is the redis sentinel master name to use to discover
         # the master for a SingleFlight lock
         MasterName = "redis-1"
diff --git a/go.mod b/go.mod
index 5d00f562a..eab26e55d 100644
--- a/go.mod
+++ b/go.mod
@@ -14,11 +14,13 @@ require (
 	github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57
 	github.com/aws/aws-sdk-go v1.15.24
 	github.com/bsm/redis-lock v8.0.0+incompatible
+	github.com/bsm/redislock v0.4.2
 	github.com/codegangsta/negroni v1.0.0 // indirect
 	github.com/fatih/color v1.7.0
 	github.com/go-playground/locales v0.12.1 // indirect
 	github.com/go-playground/universal-translator v0.16.0 // indirect
 	github.com/go-redis/redis v6.15.2+incompatible
+	github.com/go-redis/redis/v7 v7.2.0
 	github.com/gobuffalo/envy v1.6.7
 	github.com/gobuffalo/httptest v1.0.4
 	github.com/golang/snappy v0.0.1 // indirect
diff --git a/go.sum b/go.sum
index 5685aacfc..6c5029447 100644
--- a/go.sum
+++ b/go.sum
@@ -54,6 +54,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k=
 github.com/bsm/redis-lock v8.0.0+incompatible h1:QgB0J2pNG8hUfndTIvpPh38F5XsUTTvO7x8Sls++9Mk=
 github.com/bsm/redis-lock v8.0.0+incompatible/go.mod h1:8dGkQ5GimBCahwF2R67tqGCJbyDZSp0gzO7wq3pDrik=
+github.com/bsm/redislock v0.4.2 h1:+7WydoauDwf5Qw0ajaI/g3t26dQ/ovGU0Dv59sVvQzc=
+github.com/bsm/redislock v0.4.2/go.mod h1:zeuSDdDFtEDtbAgKsw7NDucfSVR0zLWBv8tMpro/6UM=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
@@ -90,6 +92,10 @@ github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rm
 github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
 github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
 github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U=
+github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
+github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs=
+github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg=
@@ -111,6 +117,8 @@ github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -209,9 +217,15 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
@@ -333,6 +347,8 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -353,6 +369,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w=
 golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -388,6 +406,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -403,6 +422,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
diff --git a/pkg/stash/with_redis.go b/pkg/stash/with_redis.go
index 2316c21b3..4b3bce2ff 100644
--- a/pkg/stash/with_redis.go
+++ b/pkg/stash/with_redis.go
@@ -4,8 +4,8 @@ import (
 	"context"
 	"time"
 
-	lock "github.com/bsm/redis-lock"
-	"github.com/go-redis/redis"
+	lock "github.com/bsm/redislock"
+	"github.com/go-redis/redis/v7"
 	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/observ"
@@ -43,17 +43,15 @@ func (s *redisLock) Stash(ctx context.Context, mod, ver string) (newVer string,
 	mv := config.FmtModVer(mod, ver)
 
 	// Obtain a new lock with default settings
-	lock, err := lock.Obtain(s.client, mv, &lock.Options{
-		LockTimeout: time.Minute * 5,
-		RetryCount:  60 * 5,
-		RetryDelay:  time.Second,
+	lock, err := lock.Obtain(s.client, mv, time.Minute*5, &lock.Options{
+		RetryStrategy: lock.LimitRetry(lock.LinearBackoff(time.Second), 60*5),
 	})
 	if err != nil {
 		return ver, errors.E(op, err)
 	}
 	defer func() {
-		const op errors.Op = "redis.Unlock"
-		lockErr := lock.Unlock()
+		const op errors.Op = "redis.Release"
+		lockErr := lock.Release()
 		if err == nil && lockErr != nil {
 			err = errors.E(op, lockErr)
 		}
diff --git a/pkg/stash/with_redis_sentinel.go b/pkg/stash/with_redis_sentinel.go
index ba23bb8a7..c53d66301 100644
--- a/pkg/stash/with_redis_sentinel.go
+++ b/pkg/stash/with_redis_sentinel.go
@@ -1,19 +1,31 @@
 package stash
 
 import (
-	"github.com/go-redis/redis"
+	errs "errors"
+
+	"github.com/go-redis/redis/v7"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/storage"
 )
 
+var (
+	// ErrNoEndpoints is returned when no endpoints were provided
+	ErrNoEndpoints = errs.New("no endpoints provided")
+)
+
 // WithRedisSentinelLock returns a distributed singleflight
 // with a redis cluster that utilizes sentinel for quorum and failover
 func WithRedisSentinelLock(endpoints []string, master, password string, checker storage.Checker) (Wrapper, error) {
 	const op errors.Op = "stash.WithRedisSentinelLock"
+	// The redis client constructor does not return an error when no endpoints
+	// are provided, so we check for ourselves.
+	if len(endpoints) == 0 {
+		return nil, ErrNoEndpoints
+	}
 	client := redis.NewFailoverClient(&redis.FailoverOptions{
-		MasterName:    master,
-		SentinelAddrs: endpoints,
-		Password:      password,
+		MasterName:       master,
+		SentinelAddrs:    endpoints,
+		SentinelPassword: password,
 	})
 	_, err := client.Ping().Result()
 	if err != nil {

From fc5e66cce7c76dabfe19e0cd6f9adcde91249a0d Mon Sep 17 00:00:00 2001
From: Ted Wexler <ted@stuckinacan.com>
Date: Mon, 24 Feb 2020 21:25:10 -0500
Subject: [PATCH 06/11] Rmoeve accidental config change

---
 config.dev.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.dev.toml b/config.dev.toml
index 0ef9b125c..07a58d16a 100755
--- a/config.dev.toml
+++ b/config.dev.toml
@@ -276,7 +276,7 @@ DownloadURL = ""
 # that only one module is ever written even when concurrent saves happen
 # at the same time.
 # Env override: ATHENS_SINGLE_FLIGHT_TYPE
-SingleFlightType = "redis-sentinel"
+SingleFlightType = "memory"
 
 [SingleFlight]
     [SingleFlight.Etcd]

From 44431c03d4d81cbe211272fa46f1d5b83c587f2a Mon Sep 17 00:00:00 2001
From: Ted Wexler <ted@stuckinacan.com>
Date: Mon, 24 Feb 2020 21:30:08 -0500
Subject: [PATCH 07/11] Fix default config

---
 pkg/config/config.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkg/config/config.go b/pkg/config/config.go
index ad108fd14..9c9262765 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -164,6 +164,11 @@ func defaultConfig() *Config {
 		SingleFlight: &SingleFlight{
 			Etcd:  &Etcd{"localhost:2379,localhost:22379,localhost:32379"},
 			Redis: &Redis{"127.0.0.1:6379"},
+			RedisSentinel: &RedisSentinel{
+				Endpoints:        []string{"127.0.0.1:26379"},
+				MasterName:       "redis-1",
+				SentinelPassword: "sekret",
+			},
 		},
 	}
 }

From 77a3c613fe2815244ed964f1990a2ec2e4ac8ed0 Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Wed, 26 Feb 2020 11:40:29 -0500
Subject: [PATCH 08/11] Addresses review comments

---
 cmd/proxy/actions/app_proxy.go   | 2 +-
 config.dev.toml                  | 6 +++++-
 pkg/stash/with_redis_sentinel.go | 9 +--------
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go
index f5b2b54c4..f1163d6d9 100644
--- a/cmd/proxy/actions/app_proxy.go
+++ b/cmd/proxy/actions/app_proxy.go
@@ -134,7 +134,7 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper,
 		}
 		return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, checker)
 	case "redis-sentinel":
-		if c.SingleFlight == nil || c.SingleFlight.Redis == nil {
+		if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil {
 			return nil, fmt.Errorf("Redis config must be present")
 		}
 		return stash.WithRedisSentinelLock(
diff --git a/config.dev.toml b/config.dev.toml
index 07a58d16a..813eae921 100755
--- a/config.dev.toml
+++ b/config.dev.toml
@@ -263,7 +263,7 @@ DownloadURL = ""
 # and the second request will wait for the first one to finish so that
 # it doesn't override the storage.
 
-# Options are ["memory", "etcd", "redis", "gcp", "azureblob"]
+# Options are ["memory", "etcd", "redis", "redis-sentinel", "gcp", "azureblob"]
 
 # The default option is "memory" which means that only one instance of Athens
 # should be used.
@@ -275,6 +275,10 @@ DownloadURL = ""
 # and therefore it will use its strong-consistency features to ensure
 # that only one module is ever written even when concurrent saves happen
 # at the same time.
+# The "redis" single flight will use a single redis instance as a locking mechanism
+# for updating the underlying storage
+# The "redis-sentinel" single flight works similarly to "redis" but obtains a redis connection
+# via a redis-sentinel
 # Env override: ATHENS_SINGLE_FLIGHT_TYPE
 SingleFlightType = "memory"
 
diff --git a/pkg/stash/with_redis_sentinel.go b/pkg/stash/with_redis_sentinel.go
index c53d66301..aa92ffee3 100644
--- a/pkg/stash/with_redis_sentinel.go
+++ b/pkg/stash/with_redis_sentinel.go
@@ -1,18 +1,11 @@
 package stash
 
 import (
-	errs "errors"
-
 	"github.com/go-redis/redis/v7"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/storage"
 )
 
-var (
-	// ErrNoEndpoints is returned when no endpoints were provided
-	ErrNoEndpoints = errs.New("no endpoints provided")
-)
-
 // WithRedisSentinelLock returns a distributed singleflight
 // with a redis cluster that utilizes sentinel for quorum and failover
 func WithRedisSentinelLock(endpoints []string, master, password string, checker storage.Checker) (Wrapper, error) {
@@ -20,7 +13,7 @@ func WithRedisSentinelLock(endpoints []string, master, password string, checker
 	// The redis client constructor does not return an error when no endpoints
 	// are provided, so we check for ourselves.
 	if len(endpoints) == 0 {
-		return nil, ErrNoEndpoints
+		return nil, errors.E(op, "no endpoints specified")
 	}
 	client := redis.NewFailoverClient(&redis.FailoverOptions{
 		MasterName:       master,

From 770a6e669e1618b022536f28b03c1a74e464e8ac Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Tue, 17 Mar 2020 10:46:47 -0400
Subject: [PATCH 09/11] Add documentation on single flight mechanisms

---
 docs/content/configuration/storage.md | 91 +++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md
index c036989d4..27a68bf1d 100644
--- a/docs/content/configuration/storage.md
+++ b/docs/content/configuration/storage.md
@@ -335,3 +335,94 @@ It assumes that you already have the following:
             # Name of container in the blob storage
             # Env override: ATHENS_AZURE_CONTAINER_NAME
             ContainerName = "MY_AZURE_BLOB_CONTAINER_NAME"
+
+## Running multiple Athens pointed at the same storage
+
+Athens has the abilit to run concurrently pointed at the same storage medium, using
+a distributed locking mechanism called "single flight".
+
+By default, Athens is configured to use the `memory` single flight, which
+stores locks in local memory. This works when running a single Athens instance, given
+the process has access to it's own memory. However, when running multiple Athens instances
+pointed at the same storage, a distributed locking mechansism is required.
+
+Athens supports several distributed locking mechanisms:
+  - `etcd`
+  - `redis`
+  - `redis-sentinel`
+  - `gcp` (available when using the `gcp` storage type)
+  - `azureblob` (available when using the `azureblob` storage type)
+
+Setting the `SingleFlightType` (or `ATHENS_SINGLE_FLIGHT TYPE` in the environment) configuration
+value will enable usage of one of the above mechanisms. The `azureblob` and `gcp` types require
+no extra configuration.
+
+### Using etcd as the single flight mechanism
+
+Using the `etcd` mechanism is very simple, just a comma separated list of etcd endpoints.
+The recommend configuration is 3 endpoints, however, more can be used.
+  
+  SingleFlightType = "etcd"
+
+  [SingleFlight]
+      [SingleFlight.Etcd]
+          # Env override: ATHENS_ETCD_ENDPOINTS
+          Endpoints = "localhost:2379,localhost:22379,localhost:32379"
+
+### Using redis as the single flight mechanism
+
+Athens supports two mechanisms of communicating with redis: direct connection, and
+connecting via redis sentinels.
+
+#### Direct connection to redis
+
+Using a direct connection to redis is simple, and only requires a single `redis-server`.
+You can also optionally specify a password to connect to the redis server with
+
+  SingleFlightType = "redis"
+
+  [SingleFlight]
+      [SingleFlight.Redis]
+          # Endpoint is the redis endpoint for the single flgiht mechanism
+          # Env override: ATHENS_REDIS_ENDPOINT
+          Endpoint = "127.0.0.1:6379"
+
+          # Password is the password for the redis instance
+          # Env override: ATHENS_REDIS_PASSWORD
+          Password = ""
+
+#### Connecting to redis via redis sentinel
+
+**NOTE**: redis-sentinel requires a working knowledge of redis and is not recommended for
+everyone.
+
+redis sentinel is a high-availability set up for redis, it provides automated monitoring, replication,
+failover and configuration of multiple redis servers in a master-follower setup. It is more
+complex than running a single redis server and requires multiple disperate instances of redis
+running distributed across nodes.
+
+For more details on redis-sentinel, check out the [documentation](https://redis.io/topics/sentinel)
+
+As redis-sentinel is a more complex set up of redis, it requires more configuration than standard redis.
+
+Required configuration:
+
+- `Endpoints` is a list of redis-sentinel endpoints to connect to, typically 3, but more can be used
+- `MasterName` is the named master instance, as configured in the `redis-sentinel` [configuration](https://redis.io/topics/sentinel#configuring-sentinel)
+
+Optionally, like `redis`, you can also specify a password to connect to the `redis-sentinel` endpoints with
+
+  SingleFlightType = "redis-sentinel"
+
+  [SingleFlight]
+    [SingleFlight.RedisSentinel]
+        # Endpoints is the redis sentinel endpoints to discover a redis 
+        # master for a SingleFlight lock.
+        # Env override: ATHENS_REDIS_SENTINEL_ENDPOINTS
+        Endpoints = ["127.0.0.1:26379"]
+        # MasterName is the redis sentinel master name to use to discover
+        # the master for a SingleFlight lock
+        MasterName = "redis-1"
+        # SentinelPassword is an optional password for authenticating with
+        # redis sentinel
+        SentinelPassword = "sekret"

From cdbdad9d091618278789cf2d1d5951ccca9e2b20 Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Tue, 17 Mar 2020 11:45:59 -0400
Subject: [PATCH 10/11] Fix spelling issues

---
 docs/content/configuration/storage.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md
index 27a68bf1d..63ea0370f 100644
--- a/docs/content/configuration/storage.md
+++ b/docs/content/configuration/storage.md
@@ -338,7 +338,7 @@ It assumes that you already have the following:
 
 ## Running multiple Athens pointed at the same storage
 
-Athens has the abilit to run concurrently pointed at the same storage medium, using
+Athens has the ability to run concurrently pointed at the same storage medium, using
 a distributed locking mechanism called "single flight".
 
 By default, Athens is configured to use the `memory` single flight, which
@@ -383,7 +383,7 @@ You can also optionally specify a password to connect to the redis server with
 
   [SingleFlight]
       [SingleFlight.Redis]
-          # Endpoint is the redis endpoint for the single flgiht mechanism
+          # Endpoint is the redis endpoint for the single flight mechanism
           # Env override: ATHENS_REDIS_ENDPOINT
           Endpoint = "127.0.0.1:6379"
 
@@ -397,7 +397,7 @@ You can also optionally specify a password to connect to the redis server with
 everyone.
 
 redis sentinel is a high-availability set up for redis, it provides automated monitoring, replication,
-failover and configuration of multiple redis servers in a master-follower setup. It is more
+failover and configuration of multiple redis servers in a leader-follower setup. It is more
 complex than running a single redis server and requires multiple disperate instances of redis
 running distributed across nodes.
 

From eea12c4ebb04173c16b7a8e2b127d27f74944d0d Mon Sep 17 00:00:00 2001
From: Ted Wexler <twexler@bloomberg.net>
Date: Tue, 17 Mar 2020 11:52:36 -0400
Subject: [PATCH 11/11] Fix formatting

---
 docs/content/configuration/storage.md | 67 ++++++++++++++-------------
 1 file changed, 34 insertions(+), 33 deletions(-)

diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md
index 63ea0370f..4525d5365 100644
--- a/docs/content/configuration/storage.md
+++ b/docs/content/configuration/storage.md
@@ -347,11 +347,12 @@ the process has access to it's own memory. However, when running multiple Athens
 pointed at the same storage, a distributed locking mechansism is required.
 
 Athens supports several distributed locking mechanisms:
-  - `etcd`
-  - `redis`
-  - `redis-sentinel`
-  - `gcp` (available when using the `gcp` storage type)
-  - `azureblob` (available when using the `azureblob` storage type)
+
+- `etcd`
+- `redis`
+- `redis-sentinel`
+- `gcp` (available when using the `gcp` storage type)
+- `azureblob` (available when using the `azureblob` storage type)
 
 Setting the `SingleFlightType` (or `ATHENS_SINGLE_FLIGHT TYPE` in the environment) configuration
 value will enable usage of one of the above mechanisms. The `azureblob` and `gcp` types require
@@ -362,12 +363,12 @@ no extra configuration.
 Using the `etcd` mechanism is very simple, just a comma separated list of etcd endpoints.
 The recommend configuration is 3 endpoints, however, more can be used.
   
-  SingleFlightType = "etcd"
+    SingleFlightType = "etcd"
 
-  [SingleFlight]
-      [SingleFlight.Etcd]
-          # Env override: ATHENS_ETCD_ENDPOINTS
-          Endpoints = "localhost:2379,localhost:22379,localhost:32379"
+    [SingleFlight]
+        [SingleFlight.Etcd]
+            # Env override: ATHENS_ETCD_ENDPOINTS
+            Endpoints = "localhost:2379,localhost:22379,localhost:32379"
 
 ### Using redis as the single flight mechanism
 
@@ -379,17 +380,17 @@ connecting via redis sentinels.
 Using a direct connection to redis is simple, and only requires a single `redis-server`.
 You can also optionally specify a password to connect to the redis server with
 
-  SingleFlightType = "redis"
+    SingleFlightType = "redis"
 
-  [SingleFlight]
-      [SingleFlight.Redis]
-          # Endpoint is the redis endpoint for the single flight mechanism
-          # Env override: ATHENS_REDIS_ENDPOINT
-          Endpoint = "127.0.0.1:6379"
+    [SingleFlight]
+        [SingleFlight.Redis]
+            # Endpoint is the redis endpoint for the single flight mechanism
+            # Env override: ATHENS_REDIS_ENDPOINT
+            Endpoint = "127.0.0.1:6379"
 
-          # Password is the password for the redis instance
-          # Env override: ATHENS_REDIS_PASSWORD
-          Password = ""
+            # Password is the password for the redis instance
+            # Env override: ATHENS_REDIS_PASSWORD
+            Password = ""
 
 #### Connecting to redis via redis sentinel
 
@@ -412,17 +413,17 @@ Required configuration:
 
 Optionally, like `redis`, you can also specify a password to connect to the `redis-sentinel` endpoints with
 
-  SingleFlightType = "redis-sentinel"
-
-  [SingleFlight]
-    [SingleFlight.RedisSentinel]
-        # Endpoints is the redis sentinel endpoints to discover a redis 
-        # master for a SingleFlight lock.
-        # Env override: ATHENS_REDIS_SENTINEL_ENDPOINTS
-        Endpoints = ["127.0.0.1:26379"]
-        # MasterName is the redis sentinel master name to use to discover
-        # the master for a SingleFlight lock
-        MasterName = "redis-1"
-        # SentinelPassword is an optional password for authenticating with
-        # redis sentinel
-        SentinelPassword = "sekret"
+    SingleFlightType = "redis-sentinel"
+
+    [SingleFlight]
+      [SingleFlight.RedisSentinel]
+          # Endpoints is the redis sentinel endpoints to discover a redis
+          # master for a SingleFlight lock.
+          # Env override: ATHENS_REDIS_SENTINEL_ENDPOINTS
+          Endpoints = ["127.0.0.1:26379"]
+          # MasterName is the redis sentinel master name to use to discover
+          # the master for a SingleFlight lock
+          MasterName = "redis-1"
+          # SentinelPassword is an optional password for authenticating with
+          # redis sentinel
+          SentinelPassword = "sekret"