Skip to content

Commit 4fa5c50

Browse files
authored
Add runtime fields/mappings (#1528)
This commit adds runtime fields/mappings to the client. See https://www.elastic.co/guide/en/elasticsearch/reference/7.14/runtime.html for details. Close #1527
1 parent cacca0b commit 4fa5c50

12 files changed

+348
-26
lines changed

.github/workflows/codeql-v7.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
codeql:
1515
strategy:
1616
matrix:
17-
go: [1.15.x, 1.16.x]
17+
go: [1.16.x, 1.17.x]
1818
os: [ubuntu-latest]
1919
name: Run ${{ matrix.go }} on ${{ matrix.os }}
2020
runs-on: ${{ matrix.os }}

.github/workflows/test-v7.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
test:
1111
strategy:
1212
matrix:
13-
go: [1.15.x, 1.16.x]
13+
go: [1.16.x, 1.17.x]
1414
os: [ubuntu-latest]
1515
name: Run ${{ matrix.go }} on ${{ matrix.os }}
1616
runs-on: ${{ matrix.os }}

docker-compose.cluster.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
services:
22
es1:
3-
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
3+
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
44
hostname: es1
55
environment:
66
- bootstrap.memory_lock=true
@@ -26,7 +26,7 @@ services:
2626
- 9200:9200
2727

2828
es2:
29-
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
29+
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
3030
hostname: es2
3131
environment:
3232
- bootstrap.memory_lock=true
@@ -52,7 +52,7 @@ services:
5252
- 9201:9200
5353

5454
es3:
55-
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
55+
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
5656
hostname: es3
5757
environment:
5858
- bootstrap.memory_lock=true

docker-compose.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
services:
22
elasticsearch:
3-
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
3+
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
44
hostname: elasticsearch
55
environment:
66
- cluster.name=elasticsearch
@@ -28,7 +28,7 @@ services:
2828
ports:
2929
- 9200:9200
3030
platinum:
31-
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
31+
image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
3232
hostname: elasticsearch-platinum
3333
environment:
3434
- cluster.name=platinum

field_caps.go

+23-9
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ type FieldCapsService struct {
3232
expandWildcards string
3333
fields []string
3434
ignoreUnavailable *bool
35+
includeUnmapped *bool
3536
bodyJson interface{}
3637
bodyString string
3738
}
@@ -117,6 +118,12 @@ func (s *FieldCapsService) IgnoreUnavailable(ignoreUnavailable bool) *FieldCapsS
117118
return s
118119
}
119120

121+
// IncludeUnmapped specifies whether unmapped fields whould be included in the response.
122+
func (s *FieldCapsService) IncludeUnmapped(includeUnmapped bool) *FieldCapsService {
123+
s.includeUnmapped = &includeUnmapped
124+
return s
125+
}
126+
120127
// BodyJson is documented as: Field json objects containing the name and optionally a range to filter out indices result, that have results outside the defined bounds.
121128
func (s *FieldCapsService) BodyJson(body interface{}) *FieldCapsService {
122129
s.bodyJson = body
@@ -160,7 +167,7 @@ func (s *FieldCapsService) buildURL() (string, url.Values, error) {
160167
params.Set("filter_path", strings.Join(s.filterPath, ","))
161168
}
162169
if s.allowNoIndices != nil {
163-
params.Set("allow_no_indices", fmt.Sprintf("%v", *s.allowNoIndices))
170+
params.Set("allow_no_indices", fmt.Sprint(*s.allowNoIndices))
164171
}
165172
if s.expandWildcards != "" {
166173
params.Set("expand_wildcards", s.expandWildcards)
@@ -169,7 +176,10 @@ func (s *FieldCapsService) buildURL() (string, url.Values, error) {
169176
params.Set("fields", strings.Join(s.fields, ","))
170177
}
171178
if s.ignoreUnavailable != nil {
172-
params.Set("ignore_unavailable", fmt.Sprintf("%v", *s.ignoreUnavailable))
179+
params.Set("ignore_unavailable", fmt.Sprint(*s.ignoreUnavailable))
180+
}
181+
if s.includeUnmapped != nil {
182+
params.Set("include_unmapped", fmt.Sprint(*s.includeUnmapped))
173183
}
174184
return path, params, nil
175185
}
@@ -231,7 +241,9 @@ func (s *FieldCapsService) Do(ctx context.Context) (*FieldCapsResponse, error) {
231241
// FieldCapsRequest can be used to set up the body to be used in the
232242
// Field Capabilities API.
233243
type FieldCapsRequest struct {
234-
Fields []string `json:"fields"`
244+
Fields []string `json:"fields"` // list of fields to retrieve
245+
IndexFilter Query `json:"index_filter,omitempty"`
246+
RuntimeMappings RuntimeMappings `json:"runtime_mappings,omitempty"`
235247
}
236248

237249
// -- Response --
@@ -248,10 +260,12 @@ type FieldCapsType map[string]FieldCaps // type -> caps
248260

249261
// FieldCaps contains capabilities of an individual field.
250262
type FieldCaps struct {
251-
Type string `json:"type"`
252-
Searchable bool `json:"searchable"`
253-
Aggregatable bool `json:"aggregatable"`
254-
Indices []string `json:"indices,omitempty"`
255-
NonSearchableIndices []string `json:"non_searchable_indices,omitempty"`
256-
NonAggregatableIndices []string `json:"non_aggregatable_indices,omitempty"`
263+
Type string `json:"type"`
264+
MetadataField bool `json:"metadata_field"`
265+
Searchable bool `json:"searchable"`
266+
Aggregatable bool `json:"aggregatable"`
267+
Indices []string `json:"indices,omitempty"`
268+
NonSearchableIndices []string `json:"non_searchable_indices,omitempty"`
269+
NonAggregatableIndices []string `json:"non_aggregatable_indices,omitempty"`
270+
Meta map[string]interface{} `json:"meta,omitempty"`
257271
}

field_caps_test.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -96,25 +96,25 @@ func TestFieldCapsResponse(t *testing.T) {
9696
"failed": 0
9797
},
9898
"fields": {
99-
"rating": {
99+
"rating": {
100100
"long": {
101101
"searchable": true,
102102
"aggregatable": false,
103103
"indices": ["index1", "index2"],
104-
"non_aggregatable_indices": ["index1"]
104+
"non_aggregatable_indices": ["index1"]
105105
},
106106
"keyword": {
107107
"searchable": false,
108108
"aggregatable": true,
109109
"indices": ["index3", "index4"],
110-
"non_searchable_indices": ["index4"]
110+
"non_searchable_indices": ["index4"]
111111
}
112112
},
113-
"title": {
113+
"title": {
114114
"text": {
115115
"searchable": true,
116116
"aggregatable": false
117-
117+
118118
}
119119
}
120120
}

runtime_mappings.go

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright 2012-present Oliver Eilhard. All rights reserved.
2+
// Use of this source code is governed by a MIT-license.
3+
// See http://olivere.mit-license.org/license.txt for details.
4+
5+
package elastic
6+
7+
// RuntimeMappings specify fields that are evaluated at query time.
8+
//
9+
// See https://www.elastic.co/guide/en/elasticsearch/reference/7.14/runtime.html
10+
// for details.
11+
type RuntimeMappings map[string]interface{}
12+
13+
// Source deserializes the runtime mappings.
14+
func (m *RuntimeMappings) Source() (interface{}, error) {
15+
return m, nil
16+
}

runtime_mappings_test.go

+148
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Copyright 2012-present Oliver Eilhard. All rights reserved.
2+
// Use of this source code is governed by a MIT-license.
3+
// See http://olivere.mit-license.org/license.txt for details.
4+
5+
package elastic
6+
7+
import (
8+
"context"
9+
"encoding/json"
10+
"testing"
11+
"time"
12+
)
13+
14+
func TestRuntimeMappingsSource(t *testing.T) {
15+
rm := RuntimeMappings{
16+
"day_of_week": map[string]interface{}{
17+
"type": "keyword",
18+
},
19+
}
20+
src, err := rm.Source()
21+
if err != nil {
22+
t.Fatal(err)
23+
}
24+
data, err := json.Marshal(src)
25+
if err != nil {
26+
t.Fatal(err)
27+
}
28+
expected := `{"day_of_week":{"type":"keyword"}}`
29+
if want, have := expected, string(data); want != have {
30+
t.Fatalf("want %s, have %s", want, have)
31+
}
32+
}
33+
34+
func TestRuntimeMappings(t *testing.T) {
35+
client := setupTestClient(t) //, SetTraceLog(log.New(os.Stdout, "", 0)))
36+
37+
ctx := context.Background()
38+
indexName := testIndexName
39+
40+
// Create index
41+
createIndex, err := client.CreateIndex(indexName).Do(ctx)
42+
if err != nil {
43+
t.Fatal(err)
44+
}
45+
if createIndex == nil {
46+
t.Errorf("expected result to be != nil; got: %v", createIndex)
47+
}
48+
49+
mapping := `{
50+
"dynamic": "runtime",
51+
"properties": {
52+
"@timestamp": {
53+
"type":"date"
54+
}
55+
},
56+
"runtime": {
57+
"day_of_week": {
58+
"type": "keyword",
59+
"script": {
60+
"source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
61+
}
62+
}
63+
}
64+
}`
65+
type Doc struct {
66+
Timestamp time.Time `json:"@timestamp"`
67+
}
68+
type DynamicDoc struct {
69+
Timestamp time.Time `json:"@timestamp"`
70+
DayOfWeek string `json:"day_of_week"`
71+
}
72+
73+
// Create mapping
74+
putResp, err := client.PutMapping().
75+
Index(indexName).
76+
BodyString(mapping).
77+
Do(ctx)
78+
if err != nil {
79+
t.Fatalf("expected put mapping to succeed; got: %v", err)
80+
}
81+
if putResp == nil {
82+
t.Fatalf("expected put mapping response; got: %v", putResp)
83+
}
84+
if !putResp.Acknowledged {
85+
t.Fatalf("expected put mapping ack; got: %v", putResp.Acknowledged)
86+
}
87+
88+
// Add a document
89+
timestamp := time.Date(2021, 1, 17, 23, 24, 25, 26, time.UTC)
90+
indexResult, err := client.Index().
91+
Index(indexName).
92+
Id("1").
93+
BodyJson(&Doc{
94+
Timestamp: timestamp,
95+
}).
96+
Refresh("wait_for").
97+
Do(ctx)
98+
if err != nil {
99+
t.Fatal(err)
100+
}
101+
if indexResult == nil {
102+
t.Errorf("expected result to be != nil; got: %v", indexResult)
103+
}
104+
105+
// Execute a search to check for runtime fields
106+
searchResp, err := client.Search(indexName).
107+
Query(NewMatchAllQuery()).
108+
DocvalueFields("@timestamp", "day_of_week").
109+
Do(ctx)
110+
if err != nil {
111+
t.Fatal(err)
112+
}
113+
if searchResp == nil {
114+
t.Errorf("expected result to be != nil; got: %v", searchResp)
115+
}
116+
if want, have := int64(1), searchResp.TotalHits(); want != have {
117+
t.Fatalf("expected %d search hits, got %d", want, have)
118+
}
119+
120+
// The hit should not have the "day_of_week"
121+
hit := searchResp.Hits.Hits[0]
122+
var doc DynamicDoc
123+
if err := json.Unmarshal(hit.Source, &doc); err != nil {
124+
t.Fatalf("unable to deserialize hit: %v", err)
125+
}
126+
if want, have := timestamp, doc.Timestamp; want != have {
127+
t.Fatalf("expected timestamp=%v, got %v", want, have)
128+
}
129+
if want, have := "", doc.DayOfWeek; want != have {
130+
t.Fatalf("expected day_of_week=%q, got %q", want, have)
131+
}
132+
133+
// The fields should include a "day_of_week" of ["Sunday"]
134+
dayOfWeekIntfSlice, ok := hit.Fields["day_of_week"].([]interface{})
135+
if !ok {
136+
t.Fatalf("expected a slice of strings, got %T", hit.Fields["day_of_week"])
137+
}
138+
if want, have := 1, len(dayOfWeekIntfSlice); want != have {
139+
t.Fatalf("expected a slice of size %d, have %d", want, have)
140+
}
141+
dayOfWeek, ok := dayOfWeekIntfSlice[0].(string)
142+
if !ok {
143+
t.Fatalf("expected an element of string, got %T", dayOfWeekIntfSlice[0])
144+
}
145+
if want, have := "Sunday", dayOfWeek; want != have {
146+
t.Fatalf("expected day_of_week=%q, have %q", want, have)
147+
}
148+
}

search.go

+44-1
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,12 @@ func (s *SearchService) PointInTime(pointInTime *PointInTime) *SearchService {
159159
return s
160160
}
161161

162+
// RuntimeMappings specifies optional runtime mappings.
163+
func (s *SearchService) RuntimeMappings(runtimeMappings RuntimeMappings) *SearchService {
164+
s.searchSource = s.searchSource.RuntimeMappings(runtimeMappings)
165+
return s
166+
}
167+
162168
// TimeoutInMillis sets the timeout in milliseconds.
163169
func (s *SearchService) TimeoutInMillis(timeoutInMillis int) *SearchService {
164170
s.searchSource = s.searchSource.TimeoutInMillis(timeoutInMillis)
@@ -764,7 +770,7 @@ type SearchHit struct {
764770
Sort []interface{} `json:"sort,omitempty"` // sort information
765771
Highlight SearchHitHighlight `json:"highlight,omitempty"` // highlighter information
766772
Source json.RawMessage `json:"_source,omitempty"` // stored document source
767-
Fields map[string]interface{} `json:"fields,omitempty"` // returned (stored) fields
773+
Fields SearchHitFields `json:"fields,omitempty"` // returned (stored) fields
768774
Explanation *SearchExplanation `json:"_explanation,omitempty"` // explains how the score was computed
769775
MatchedQueries []string `json:"matched_queries,omitempty"` // matched queries
770776
InnerHits map[string]*SearchHitInnerHits `json:"inner_hits,omitempty"` // inner hits with ES >= 1.5.0
@@ -777,6 +783,43 @@ type SearchHit struct {
777783
// MatchedFilters
778784
}
779785

786+
// SearchHitFields helps to simplify resolving slices of specific types.
787+
type SearchHitFields map[string]interface{}
788+
789+
// Strings returns a slice of strings for the given field, if there is any
790+
// such field in the hit. The method ignores elements that are not of type
791+
// string.
792+
func (f SearchHitFields) Strings(fieldName string) ([]string, bool) {
793+
slice, ok := f[fieldName].([]interface{})
794+
if !ok {
795+
return nil, false
796+
}
797+
results := make([]string, 0, len(slice))
798+
for _, item := range slice {
799+
if v, ok := item.(string); ok {
800+
results = append(results, v)
801+
}
802+
}
803+
return results, true
804+
}
805+
806+
// Float64s returns a slice of float64's for the given field, if there is any
807+
// such field in the hit. The method ignores elements that are not of
808+
// type float64.
809+
func (f SearchHitFields) Float64s(fieldName string) ([]float64, bool) {
810+
slice, ok := f[fieldName].([]interface{})
811+
if !ok {
812+
return nil, false
813+
}
814+
results := make([]float64, 0, len(slice))
815+
for _, item := range slice {
816+
if v, ok := item.(float64); ok {
817+
results = append(results, v)
818+
}
819+
}
820+
return results, true
821+
}
822+
780823
// SearchHitInnerHits is used for inner hits.
781824
type SearchHitInnerHits struct {
782825
Hits *SearchHits `json:"hits,omitempty"`

0 commit comments

Comments
 (0)