Skip to content

Commit 94e2e75

Browse files
MB-66396: Index insights - TermFrequencies, CentroidCardinalities
Requires: - blevesearch/faiss#54 - blevesearch/bleve#2207 REST endpoints: - POST /api/index/{indexName}/insights - POST /api/bucket/{bucketName}/scope/{scopeName}/index/{indexName}/insights - RequestBody: `{"field": "<fieldName>", "insight": "termFrequencies", "limit": 5, "descending": false}` `{"field": "<fieldName>", "insight": "centroidCardinalities", "limit": 5, "descending": true}` - Scatter gather request uses RESTful IndexClient(s). Change-Id: Ib9b7782e3fe80f0e6bf93905e5a5f1bb76d8fb71 Reviewed-on: https://review.couchbase.org/c/cbft/+/232479 Tested-by: Abhi Dangeti <abhinav@couchbase.com> Well-Formed: Build Bot <build@couchbase.com> Reviewed-by: Rahul Rampure <rahul.rampure@couchbase.com>
1 parent d458a46 commit 94e2e75

10 files changed

Lines changed: 318 additions & 21 deletions

File tree

cache_bleve.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,3 +349,19 @@ func (m *cacheBleveIndex) HandleTask(req []byte) (
349349

350350
return nil, fmt.Errorf("bleve: unknown task op: %s", task.Op)
351351
}
352+
353+
func (m *cacheBleveIndex) TermFrequencies(field string, limit int, descending bool) (
354+
[]index.TermFreq, error) {
355+
if insightsIdx, ok := m.bindex.(bleve.InsightsIndex); ok {
356+
return insightsIdx.TermFrequencies(field, limit, descending)
357+
}
358+
return nil, nil
359+
}
360+
361+
func (m *cacheBleveIndex) CentroidCardinalities(field string, limit int, descending bool) (
362+
[]index.CentroidCardinality, error) {
363+
if insightsIdx, ok := m.bindex.(bleve.InsightsIndex); ok {
364+
return insightsIdx.CentroidCardinalities(field, limit, descending)
365+
}
366+
return nil, nil
367+
}

cmd/cbft/main.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -636,7 +636,6 @@ func mainStart(cfg cbgt.Cfg, uuid string, tags []string, container string,
636636

637637
handle(prefix+"/api/bucket/{bucketName}/scope/{scopeName}/index/{indexName}/analyzeDoc", "POST",
638638
cbft.NewAnalyzeDocHandler(mgr))
639-
640639
handle(prefix+"/api/index/{indexName}/analyzeDoc", "POST",
641640
cbft.NewAnalyzeDocHandler(mgr))
642641

@@ -660,6 +659,12 @@ func mainStart(cfg cbgt.Cfg, uuid string, tags []string, container string,
660659

661660
handle(prefix+"/api/conciseOptions", "GET", cbft.NewConciseOptions(mgr))
662661

662+
// Index insights handlers
663+
handle(prefix+"/api/bucket/{bucketName}/scope/{scopeName}/index/{indexName}/insights", "POST",
664+
cbft.NewIndexInsightsHandler(mgr))
665+
handle(prefix+"/api/index/{indexName}/insights", "POST",
666+
cbft.NewIndexInsightsHandler(mgr))
667+
663668
router := exportMuxRoutesToHttprouter(muxrouter, options)
664669

665670
router.Handler("PUT", prefix+"/api/managerOptions",

go.mod

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ require (
88
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
99
github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
1010
github.com/blevesearch/bleve-mapping-ui v0.7.0
11-
github.com/blevesearch/bleve/v2 v2.5.4
12-
github.com/blevesearch/bleve_index_api v1.2.10
11+
github.com/blevesearch/bleve/v2 v2.5.5-0.20251107200128-c79ff1534d2d
12+
github.com/blevesearch/bleve_index_api v1.2.11
1313
github.com/blevesearch/zapx/v11 v11.4.2
1414
github.com/blevesearch/zapx/v12 v12.4.2
1515
github.com/blevesearch/zapx/v13 v13.4.2
1616
github.com/blevesearch/zapx/v14 v14.4.2
1717
github.com/blevesearch/zapx/v15 v15.4.2
18-
github.com/blevesearch/zapx/v16 v16.2.6
18+
github.com/blevesearch/zapx/v16 v16.2.7
1919
github.com/buger/jsonparser v1.1.1
2020
github.com/cloudfoundry/gosigar v1.3.4
2121
github.com/couchbase/cbauth v0.1.13
@@ -56,12 +56,12 @@ require (
5656
github.com/beorn7/perks v1.0.1 // indirect
5757
github.com/bits-and-blooms/bitset v1.22.0 // indirect
5858
github.com/blevesearch/geo v0.2.4 // indirect
59-
github.com/blevesearch/go-faiss v1.0.25 // indirect
59+
github.com/blevesearch/go-faiss v1.0.26 // indirect
6060
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
6161
github.com/blevesearch/goleveldb v1.0.1 // indirect
6262
github.com/blevesearch/gtreap v0.1.1 // indirect
6363
github.com/blevesearch/mmap-go v1.0.4 // indirect
64-
github.com/blevesearch/scorch_segment_api/v2 v2.3.12 // indirect
64+
github.com/blevesearch/scorch_segment_api/v2 v2.3.13 // indirect
6565
github.com/blevesearch/segment v0.9.1 // indirect
6666
github.com/blevesearch/snowballstem v0.9.0 // indirect
6767
github.com/blevesearch/stempel v0.2.0 // indirect

go.sum

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,14 @@ github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCk
5454
github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
5555
github.com/blevesearch/bleve-mapping-ui v0.7.0 h1:gcrRWX5vjie2JEu5lgnn6tcwagcDEbD6d9XFkevt4fY=
5656
github.com/blevesearch/bleve-mapping-ui v0.7.0/go.mod h1:HeY/8LvxDgT8Qt7iqfw0dodA3Z8wAbr1hPgCnQIL5Ec=
57-
github.com/blevesearch/bleve/v2 v2.5.4 h1:1iur8e+PHsxtncV2xIVuqlQme/V8guEDO2uV6Wll3lQ=
58-
github.com/blevesearch/bleve/v2 v2.5.4/go.mod h1:yB4PnV4N2q5rTEpB2ndG8N2ISexBQEFIYgwx4ztfvoo=
59-
github.com/blevesearch/bleve_index_api v1.2.10 h1:FMFmZCmTX6PdoLLvwUnKF2RsmILFFwO3h0WPevXY9fE=
60-
github.com/blevesearch/bleve_index_api v1.2.10/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
57+
github.com/blevesearch/bleve/v2 v2.5.5-0.20251107200128-c79ff1534d2d h1:bUH0Kq4pRvQPeFcfyF07Xv1UgGYe7ffdC8JrNjtzqQ4=
58+
github.com/blevesearch/bleve/v2 v2.5.5-0.20251107200128-c79ff1534d2d/go.mod h1:t5WoESS5TDteTdnjhhvpA1BpLYErOBX2IQViTMLK7wo=
59+
github.com/blevesearch/bleve_index_api v1.2.11 h1:bXQ54kVuwP8hdrXUSOnvTQfgK0KI1+f9A0ITJT8tX1s=
60+
github.com/blevesearch/bleve_index_api v1.2.11/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
6161
github.com/blevesearch/geo v0.2.4 h1:ECIGQhw+QALCZaDcogRTNSJYQXRtC8/m8IKiA706cqk=
6262
github.com/blevesearch/geo v0.2.4/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8=
63-
github.com/blevesearch/go-faiss v1.0.25 h1:lel1rkOUGbT1CJ0YgzKwC7k+XH0XVBHnCVWahdCXk4U=
64-
github.com/blevesearch/go-faiss v1.0.25/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
63+
github.com/blevesearch/go-faiss v1.0.26 h1:4dRLolFgjPyjkaXwff4NfbZFdE/dfywbzDqporeQvXI=
64+
github.com/blevesearch/go-faiss v1.0.26/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
6565
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
6666
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
6767
github.com/blevesearch/goleveldb v1.0.1 h1:iAtV2Cu5s0GD1lwUiekkFHe2gTMCCNVj2foPclDLIFI=
@@ -71,8 +71,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
7171
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
7272
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
7373
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
74-
github.com/blevesearch/scorch_segment_api/v2 v2.3.12 h1:GGZc2qwbyRBwtckPPkHkLyXw64mmsLJxdturBI1cM+c=
75-
github.com/blevesearch/scorch_segment_api/v2 v2.3.12/go.mod h1:JBRGAneqgLSI2+jCNjtwMqp2B7EBF3/VUzgDPIU33MM=
74+
github.com/blevesearch/scorch_segment_api/v2 v2.3.13 h1:ZPjv/4VwWvHJZKeMSgScCapOy8+DdmsmRyLmSB88UoY=
75+
github.com/blevesearch/scorch_segment_api/v2 v2.3.13/go.mod h1:ENk2LClTehOuMS8XzN3UxBEErYmtwkE7MAArFTXs9Vc=
7676
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
7777
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
7878
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
@@ -93,8 +93,8 @@ github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT
9393
github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8=
9494
github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k=
9595
github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
96-
github.com/blevesearch/zapx/v16 v16.2.6 h1:OHuUl2GhM+FpBq9RwNsJ4k/QodqbMMHoQEgn/IHYpu8=
97-
github.com/blevesearch/zapx/v16 v16.2.6/go.mod h1:cuAPB+YoIyRngNhno1S1GPr9SfMk+x/SgAHBLXSIq3k=
96+
github.com/blevesearch/zapx/v16 v16.2.7 h1:xcgFRa7f/tQXOwApVq7JWgPYSlzyUMmkuYa54tMDuR0=
97+
github.com/blevesearch/zapx/v16 v16.2.7/go.mod h1:murSoCJPCk25MqURrcJaBQ1RekuqSCSfMjXH4rHyA14=
9898
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
9999
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
100100
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

missing.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,3 +127,13 @@ func (m *MissingPIndex) DeleteInternal(key []byte) error {
127127
func (m *MissingPIndex) Advanced() (index.Index, error) {
128128
return nil, missingPIndexUnimplementedErr
129129
}
130+
131+
func (m *MissingPIndex) TermFrequencies(field string, limit int, descending bool) (
132+
[]index.TermFreq, error) {
133+
return nil, nil
134+
}
135+
136+
func (m *MissingPIndex) CentroidCardinalities(field string, limit int, descending bool) (
137+
[]index.CentroidCardinality, error) {
138+
return nil, nil
139+
}

pindex_bleve.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3256,6 +3256,7 @@ func addIndexClients(mgr *cbgt.Manager, indexName, indexUUID string,
32563256
PIndexNames: []string{remotePlanPIndex.PlanPIndex.Name},
32573257
QueryURL: baseURL + "/query",
32583258
CountURL: baseURL + "/count",
3259+
InsightsURL: hostPortUrl + prefix + "/api/index/" + indexName + "/insights",
32593260
Consistency: consistencyParams,
32603261
httpClient: cbgt.HttpClient(),
32613262
}
@@ -3374,11 +3375,14 @@ func bleveIndexTargets(mgr *cbgt.Manager, indexName, indexUUID string,
33743375
}
33753376
}
33763377

3377-
remoteClients, err := rcAdder(mgr, indexName, indexUUID,
3378-
remotePlanPIndexes, consistencyParams, onlyPIndexes,
3379-
collector, groupByNode)
3380-
if err != nil {
3381-
return nil, numPIndexes, err
3378+
var remoteClients []RemoteClient
3379+
if rcAdder != nil {
3380+
remoteClients, err = rcAdder(mgr, indexName, indexUUID,
3381+
remotePlanPIndexes, consistencyParams, onlyPIndexes,
3382+
collector, groupByNode)
3383+
if err != nil {
3384+
return nil, numPIndexes, err
3385+
}
33823386
}
33833387

33843388
// TODO: Should kickoff remote queries concurrently before we wait.

pindex_impl_noop.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,13 @@ func (p *noopBleveIndex) DeleteInternal(key []byte) error {
115115
func (p *noopBleveIndex) Advanced() (index.Index, error) {
116116
return nil, noopBleveIndexUnimplementedErr
117117
}
118+
119+
func (p *noopBleveIndex) TermFrequencies(field string, limit int, descending bool) (
120+
[]index.TermFreq, error) {
121+
return nil, nil
122+
}
123+
124+
func (p *noopBleveIndex) CentroidCardinalities(field string, limit int, descending bool) (
125+
[]index.CentroidCardinality, error) {
126+
return nil, nil
127+
}

remote.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ type IndexClient struct {
9292
PIndexNames []string
9393
QueryURL string
9494
CountURL string
95+
InsightsURL string
9596
TaskRequestURL string
9697
Consistency *cbgt.ConsistencyParams
9798
httpClient cbgt.HTTPClient
@@ -427,6 +428,7 @@ func GroupIndexClientsByHostPort(clients []*IndexClient) (rv []*IndexClient, err
427428
IndexUUID: client.IndexUUID,
428429
QueryURL: baseURL + "/query",
429430
CountURL: baseURL + "/count",
431+
InsightsURL: baseURL + "/insights",
430432
TaskRequestURL: baseURL + "/tasks",
431433
Consistency: client.Consistency,
432434
httpClient: client.httpClient,
@@ -516,3 +518,90 @@ func completeTaskStatus(req *cbgt.TaskRequest, err error,
516518
}
517519
return rv
518520
}
521+
522+
func (r *IndexClient) fetchInsight(
523+
insight string,
524+
field string,
525+
limit int,
526+
descending bool,
527+
out any,
528+
) error {
529+
u, err := UrlWithAuth(r.AuthType(), r.InsightsURL)
530+
if err != nil {
531+
return fmt.Errorf("remote: insights auth for query, insightsURL: %s,"+
532+
" authType: %s, err: %w", r.InsightsURL, r.AuthType(), err)
533+
}
534+
535+
body, err := MarshalJSON(map[string]any{
536+
"field": field,
537+
"insight": insight,
538+
"limit": limit,
539+
"descending": descending,
540+
"local-only": true,
541+
})
542+
if err != nil {
543+
return fmt.Errorf("remote: insights marshal request, err: %w", err)
544+
}
545+
546+
req, err := http.NewRequest("POST", u, bytes.NewReader(body))
547+
if err != nil {
548+
return fmt.Errorf("remote: insights new request, err: %w", err)
549+
}
550+
req.Header.Set("Content-Type", "application/json")
551+
552+
resp, err := r.httpClient.Do(req)
553+
if err != nil {
554+
return fmt.Errorf("remote: insights http request, err: %w", err)
555+
}
556+
defer resp.Body.Close()
557+
558+
if resp.StatusCode != http.StatusOK {
559+
return fmt.Errorf("remote: insights got status code %d, insightsURL: %s",
560+
resp.StatusCode, r.InsightsURL)
561+
}
562+
563+
data, err := io.ReadAll(resp.Body)
564+
if err != nil {
565+
return fmt.Errorf("remote: insights read failure, err: %w", err)
566+
}
567+
568+
if err := UnmarshalJSON(data, out); err != nil {
569+
return fmt.Errorf("remote: insights parse response, err: %w", err)
570+
}
571+
572+
return nil
573+
}
574+
575+
func (r *IndexClient) TermFrequencies(field string, limit int, descending bool) (
576+
[]index.TermFreq, error) {
577+
var rv struct {
578+
Status string `json:"status"`
579+
TermFrequencies []index.TermFreq `json:"termFrequencies"`
580+
}
581+
582+
if err := r.fetchInsight(
583+
"termFrequencies", field, limit, descending, &rv); err != nil {
584+
return nil, err
585+
}
586+
if rv.Status != "ok" {
587+
return nil, fmt.Errorf("remote: insights unsupported")
588+
}
589+
return rv.TermFrequencies, nil
590+
}
591+
592+
func (r *IndexClient) CentroidCardinalities(field string, limit int, descending bool) (
593+
[]index.CentroidCardinality, error) {
594+
var rv struct {
595+
Status string `json:"status"`
596+
CentroidCardinalities []index.CentroidCardinality `json:"centroidCardinalities"`
597+
}
598+
599+
if err := r.fetchInsight(
600+
"centroidCardinalities", field, limit, descending, &rv); err != nil {
601+
return nil, err
602+
}
603+
if rv.Status != "ok" {
604+
return nil, fmt.Errorf("remote: insights unsupported")
605+
}
606+
return rv.CentroidCardinalities, nil
607+
}

knn_nosup.go renamed to vectors_nosup.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,12 @@ package cbft
1313

1414
import (
1515
"encoding/json"
16+
"net/http"
1617

1718
"github.com/blevesearch/bleve/v2"
19+
20+
"github.com/couchbase/cbgt"
21+
"github.com/couchbase/cbgt/rest"
1822
)
1923

2024
const featuresVectorBase64Dims4096 = ""
@@ -66,3 +70,17 @@ func GetKNNThrottleLimit() int64 {
6670

6771
func SetKNNThrottleLimit(val int64) {
6872
}
73+
74+
// -----------------------------------------------------------------------------
75+
76+
type IndexInsightsHandler struct {
77+
}
78+
79+
func NewIndexInsightsHandler(mgr *cbgt.Manager) *IndexInsightsHandler {
80+
return &IndexInsightsHandler{}
81+
}
82+
83+
func (h *IndexInsightsHandler) ServeHTTP(
84+
w http.ResponseWriter, req *http.Request) {
85+
rest.ShowError(w, req, "Not supported on cluster", http.StatusNotImplemented)
86+
}

0 commit comments

Comments
 (0)