prometheus: GetRef returns 0 for existing series

The following call to GetRef always returns 0 in the first iteration even when the series exists. The strange thing is that when is commented out labels.Label{Name: "domain", Value: "b"}, it returns the existing reference for the series as expected.

This is with the current commit on the main branch #03b354d4d9386e4b3bfbcd45da4bb58b182051a5

I am not sure if this is a bug or the implementation of using GetRef is incorrect.

cc @codesome , @tomwilkie , @bboreham

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/pkg/timestamp"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/tsdb"
)

func main() {
	tsdbOptions := tsdb.DefaultOptions()
	dbPath := "db"
	os.RemoveAll(dbPath)

	if err := os.MkdirAll(dbPath, 0777); err != nil {
		log.Fatal(err, "creating tsdb DB folder")
	}
	tsDB, err := tsdb.Open(dbPath, nil, nil, tsdbOptions)
	if err != nil {
		log.Fatal(err, "creating tsdb DB")
	}

	defer func() {
		if err := tsDB.Close(); err != nil {
			log.Fatal("msg", "closing the tsdb", "err", err)
		}
	}()

	lbls2 := labels.Labels{
		labels.Label{Name: "__name__", Value: "name"},
		labels.Label{Name: "source", Value: "a"},
	}

	lbls3 := labels.Labels{
		labels.Label{Name: "__name__", Value: "name"},
		labels.Label{Name: "source", Value: "a"},
		labels.Label{Name: "domain", Value: "b"},
	}

	fmt.Println("adding 3 labels first append")
	append(tsDB, lbls3)

	fmt.Println("adding 2 labels first append")
	append(tsDB, lbls2)

	if err := tsDB.Close(); err != nil {
		log.Fatal("msg", "closing the tsdb", "err", err)
	}
	fmt.Println("reopening db")
	tsDB, err = tsdb.Open(dbPath, nil, nil, tsdbOptions)
	if err != nil {
		log.Fatal(err, "creating tsdb DB")
	}

	fmt.Println("adding 3 labels second append")
	append(tsDB, lbls3)
	fmt.Println("adding 2 labels second append")
	append(tsDB, lbls2)

}

func append(tsDB *tsdb.DB, lbls labels.Labels) {
	for i := 0; i < 3; i++ {
		appender := tsDB.Appender(context.Background())

		var (
			ref          uint64
			copiedLabels labels.Labels
		)
		g, ok := appender.(storage.GetRef)
		if !ok {
			log.Fatal("msg", "storage not ref getter")

		}

		ref, copiedLabels = g.GetRef(lbls)

		if ref != 0 {
			lbls = copiedLabels
		}
		log.Println("msg", "adding value to db", "ref", ref)
		if _, err := appender.Append(ref,
			lbls,
			timestamp.FromTime(time.Now()),
			1,
		); err != nil {
			log.Fatal("msg", "append values to the DB", "err", err)
		}

		if err := appender.Commit(); err != nil {
			log.Fatal("msg", "adding values to the DB", "err", err)
		}

		time.Sleep(time.Second)
	}
}

output:

adding 3 labels first append
2021/05/27 15:04:38 msg adding value to db ref 0
2021/05/27 15:04:39 msg adding value to db ref 1
2021/05/27 15:04:40 msg adding value to db ref 1
adding 2 labels first append
2021/05/27 15:04:41 msg adding value to db ref 0
2021/05/27 15:04:42 msg adding value to db ref 2
2021/05/27 15:04:43 msg adding value to db ref 2

reopening db
adding 3 labels second append
2021/05/27 15:04:44 msg adding value to db ref 0<-first problem, why GetRef returns 0 for existing timeseries?
2021/05/27 15:04:45 msg adding value to db ref 3<-second problem why Append created a new reference for the same time series?
2021/05/27 15:04:46 msg adding value to db ref 3
adding 2 labels second append
2021/05/27 15:04:47 msg adding value to db ref 2<-correctly returns the of the time series
2021/05/27 15:04:48 msg adding value to db ref 2
2021/05/27 15:04:49 msg adding value to db ref 2