Hi,
I was able to get some of the lower level functions working together to do hnsw vector searches but I am having some trouble figuring out how to get the store working so that I can save to disk and open and close it like a proper database.
Not interesting in using the go client functions, I’d rather interact directly with the database interfaces if that is possible.
I have attached some of the init code I am using to turn on the db, but I am unsure what needs to be called to load from the same exact path after Flush() and Shutdown() VectorIndex interface functions are called.
Any assistance would be appreciated.
Thank you.
func newStore(rootDir string, logger logrus.FieldLogger) (*lsmkv.Store, error) {
if err := os.MkdirAll(rootDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create directory %s: %v", rootDir, err)
}
compactCallbacks := cyclemanager.NewCallbackGroup("compact", logger, 1)
flushCallbacks := cyclemanager.NewCallbackGroup("flush", logger, 1)
tombstoneCallbacks := cyclemanager.NewCallbackGroup("tombstone", logger, 1)
store, err := lsmkv.New(
rootDir, rootDir, logger, nil,
compactCallbacks, flushCallbacks, tombstoneCallbacks,
)
return store, err
}
func (b *BSONTable) initHNSW() error {
makeCL := func() (hnsw.CommitLogger, error) {
return hnsw.NewCommitLogger(b.hnswPath, b.Name, b.logger, cyclemanager.NewCallbackGroup("commitLoggerThunk", b.logger, 1))
}
index, err := hnsw.New(hnsw.Config{
RootPath: b.hnswPath,
ID: fmt.Sprintf("%s_%s", b.Name, b.VectorField),
MakeCommitLoggerThunk: makeCL,
DistanceProvider: distancer.NewL2SquaredProvider(),
VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) {
key := make([]byte, 8)
binary.LittleEndian.PutUint64(key, id)
data, err := b.GetRow(key)
if err != nil {
return nil, err
}
if vec, ok := data[b.VectorField].([]float32); ok {
return vec, nil
}
return nil, fmt.Errorf("vector for ID %d not found", id)
},
}, ent.UserConfig{
CleanupIntervalSeconds: 10,
VectorCacheMaxObjects: 1000000,
Distance: "l2-squared",
DynamicEFMin: 20,
DynamicEFMax: 100,
DynamicEFFactor: 8,
EFConstruction: 200,
MaxConnections: 10,
}, cyclemanager.NewCallbackGroup("cml", b.logger, 1), b.store)
if err != nil {
return err
}
b.HnswIndex = index
return nil
}
