Skip to content

Commit b8fa12c

Browse files
authored
[Transport] Use elastic-transport-go as transport (#375)
* Initial commit to use estransport as a external dependency * Adapt import name to transport package change * Rename estransport to elastictransport * Update transport reference in makefile * Update reference to elastic-transport-go * Apply external transport to examples * Rename transport in readme files * Adding go.sum for elastic-transport
1 parent 77cd45a commit b8fa12c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+960
-5572
lines changed

Makefile

+19-17
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@ endif
3434
export ELASTICSEARCH_URL='http://elastic:elastic@localhost:9200'; \
3535
echo "gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml --" $(testintegargs); \
3636
gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "."; \
37-
gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "./estransport" "./esapi" "./esutil"; \
37+
gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "./esapi" "./esutil"; \
3838
else \
3939
export ELASTICSEARCH_URL='http://elastic:elastic@localhost:9200'; \
4040
echo "go test -v" $(testintegargs) "."; \
41-
go test -v $(testintegargs) "./estransport" "./esapi" "./esutil"; \
41+
go test -v $(testintegargs) "./esapi" "./esutil"; \
4242
fi;
4343

4444
test-api: ## Run generated API integration tests
@@ -103,15 +103,17 @@ test-examples: ## Execute the _examples
103103
printf "\033[2m────────────────────────────────────────────────────────────────────────────────\n"; \
104104
printf "\033[1mUpdating dependencies for $$d\033[0m\n"; \
105105
printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
106-
(cd $$d && go mod download && make setup) || \
106+
(cd $$d && go mod download all && make setup test) || \
107107
( \
108108
printf "\033[31m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
109109
printf "\033[31;1m⨯ ERROR\033[0m\n"; \
110110
false; \
111111
); \
112112
done; \
113113
\
114-
for f in _examples/*.go; do \
114+
( \
115+
cd _examples; \
116+
for f in *.go; do \
115117
printf "\033[2m────────────────────────────────────────────────────────────────────────────────\n"; \
116118
printf "\033[1m$$f\033[0m\n"; \
117119
printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
@@ -122,18 +124,19 @@ test-examples: ## Execute the _examples
122124
false; \
123125
); \
124126
done; \
125-
\
126-
for f in _examples/*/; do \
127-
printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
128-
printf "\033[1m$$f\033[0m\n"; \
129-
printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
130-
(cd $$f && make test && true) || \
131-
( \
132-
printf "\033[31m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
133-
printf "\033[31;1m⨯ ERROR\033[0m\n"; \
134-
false; \
135-
); \
136-
done; \
127+
);\
128+
# \
129+
# for f in _examples/*/; do \
130+
# printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
131+
# printf "\033[1m$$f\033[0m\n"; \
132+
# printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
133+
# (cd $$f && make test && true) || \
134+
# ( \
135+
# printf "\033[31m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
136+
# printf "\033[31;1m⨯ ERROR\033[0m\n"; \
137+
# false; \
138+
# ); \
139+
# done; \
137140
printf "\033[32m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
138141
\
139142
printf "\033[32;1mSUCCESS\033[0m\n"; \
@@ -263,7 +266,6 @@ godoc: ## Display documentation for the package
263266
@printf "\033[2m→ Generating documentation...\033[0m\n"
264267
@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8"
265268
@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8/esapi"
266-
@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8/estransport"
267269
@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8/esutil"
268270
@printf "\n"
269271
godoc --http=localhost:6060 --play

README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ go run main.go
8484

8585
## Usage
8686

87-
The `elasticsearch` package ties together two separate packages for calling the Elasticsearch APIs and transferring data over HTTP: `esapi` and `estransport`, respectively.
87+
The `elasticsearch` package ties together two separate packages for calling the Elasticsearch APIs and transferring data over HTTP: `esapi` and `elastictransport`, respectively.
8888

8989
Use the `elasticsearch.NewDefaultClient()` function to create the client with the default settings.
9090

@@ -351,7 +351,7 @@ func main() {
351351
As you see in the example above, the `esapi` package allows to call the Elasticsearch APIs in two distinct ways: either by creating a struct, such as `IndexRequest`, and calling its `Do()` method by passing it a context and the client, or by calling the `Search()` function on the client directly, using the option functions such as `WithIndex()`. See more information and examples in the
352352
[package documentation](https://godoc.org/github.com/elastic/go-elasticsearch/esapi).
353353

354-
The `estransport` package handles the transfer of data to and from Elasticsearch, including retrying failed requests, keeping a connection pool, discovering cluster nodes and logging.
354+
The `elastictransport` package handles the transfer of data to and from Elasticsearch, including retrying failed requests, keeping a connection pool, discovering cluster nodes and logging.
355355

356356
Read more about the client internals and usage in the following blog posts:
357357

_benchmarks/benchmarks/actions/006_bulk_helper.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@ import (
2323
"fmt"
2424
"strings"
2525

26+
"github.com/elastic/elastic-transport-go/v8/elastictransport"
27+
2628
"github.com/elastic/go-elasticsearch/v8"
2729
"github.com/elastic/go-elasticsearch/v8/esapi"
28-
"github.com/elastic/go-elasticsearch/v8/estransport"
2930
"github.com/elastic/go-elasticsearch/v8/esutil"
3031

3132
"github.com/elastic/go-elasticsearch/v8/benchmarks"
@@ -75,7 +76,7 @@ func init() {
7576
)
7677

7778
var addresses []string
78-
for _, u := range c.RunnerClient.Transport.(*estransport.Client).URLs() {
79+
for _, u := range c.RunnerClient.Transport.(*elastictransport.Client).URLs() {
7980
addresses = append(addresses, u.String())
8081
}
8182

_benchmarks/benchmarks/cmd/main.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import (
3232
"github.com/montanaflynn/stats"
3333

3434
"github.com/elastic/go-elasticsearch/v8"
35-
"github.com/elastic/go-elasticsearch/v8/estransport"
35+
"github.com/elastic/elastic-transport-go/v8/elastictransport"
3636

3737
"github.com/elastic/go-elasticsearch/v8/benchmarks"
3838
"github.com/elastic/go-elasticsearch/v8/benchmarks/runner"
@@ -125,8 +125,8 @@ func main() {
125125
EnableRetryOnTimeout: true,
126126
}
127127
if os.Getenv("DEBUG") != "" {
128-
runnerClientConfig.Logger = &estransport.ColorLogger{Output: os.Stdout}
129-
reportClientConfig.Logger = &estransport.ColorLogger{Output: os.Stdout}
128+
runnerClientConfig.Logger = &elastictransport.ColorLogger{Output: os.Stdout}
129+
reportClientConfig.Logger = &elastictransport.ColorLogger{Output: os.Stdout}
130130
}
131131

132132
runnerClient, _ := elasticsearch.NewClient(runnerClientConfig)

_examples/bulk/Makefile

+1-6
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,7 @@ test-benchmarks: clean setup
1313
cd benchmarks && go run benchmarks.go
1414

1515
setup:
16-
@go get -u github.com/mailru/easyjson/...
17-
@go mod download github.com/dustin/go-humanize
18-
@go mod download github.com/cenkalti/backoff/v4
19-
@cd benchmarks && \
20-
go mod download && \
21-
go mod download github.com/mailru/easyjson && \
16+
go get -u github.com/mailru/easyjson/...
2217
go generate ./model
2318

2419
clean:

_examples/bulk/README.md

+4-3
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
The [`default.go`](default.go) example demonstrates how to properly operate the Elasticsearch's
66
[Bulk API]([https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html]).
77

8-
The example intentionally doesn't use any abstractions or helper functions, to
9-
demonstrate the low-level mechanics of working with the Bulk API:
8+
The example intentionally doesn't use any abstractions or helper functions, to demonstrate the low-level mechanics of
9+
working with the Bulk API:
1010

1111
* iterating over a slice of data and preparing the `meta`/`data` pairs,
1212
* filling a buffer with the payload until the configured threshold for a single batch is reached,
@@ -29,7 +29,8 @@ go run default.go -count=100000 -batch=25000
2929

3030
## `indexer.go`
3131

32-
The [`indexer.go`](indexer.go) example demonstrates how to use the [`esutil.BulkIndexer`](../esutil/bulk_indexer.go) helper for efficient indexing in parallel.
32+
The [`indexer.go`](indexer.go) example demonstrates how to use the [`esutil.BulkIndexer`](../esutil/bulk_indexer.go)
33+
helper for efficient indexing in parallel.
3334

3435
```bash
3536
go run indexer.go -count=100000 -flush=1000000

_examples/bulk/benchmarks/README.md

+15-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# Bulk Indexer Benchmarks
22

3-
The [`benchmarks.go`](benchmarks.go) file executes end-to-end benchmarks for `esutil.NewBulkIndexer`. It allows to configure indexer parameters, index settings, number of runs. See `go run benchmarks.go --help` for an overview of configuration options:
3+
The [`benchmarks.go`](benchmarks.go) file executes end-to-end benchmarks for `esutil.NewBulkIndexer`. It allows to
4+
configure indexer parameters, index settings, number of runs. See `go run benchmarks.go --help` for an overview of
5+
configuration options:
46

57
```
68
go run benchmarks.go --help
@@ -67,7 +69,8 @@ docs/sec: min [279,173] max [289,351] mean [286,987]
6769

6870
## HTTP Log Event
6971

70-
The [`httplog`](data/httplog/document.json) dataset uses a bigger document (2.5K), corresponding to a log event gathered by [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-nginx.html) from Nginx.
72+
The [`httplog`](data/httplog/document.json) dataset uses a bigger document (2.5K), corresponding to a log event gathered
73+
by [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-nginx.html) from Nginx.
7174

7275
```
7376
ELASTICSEARCH_URL=http://server:9200 go run benchmarks.go --dataset=httplog --count=1_000_000 --flush=3MB --shards=5 --replicas=0 --fasthttp=true --easyjson=true
@@ -89,7 +92,8 @@ docs/sec: min [50,165] max [53,072] mean [52,011]
8992

9093
## Mock Server
9194

92-
The `--mockserver` flag allows to run the benchmark against a "mock server", in this case Nginx, to understand a theoretical performance of the client, without the overhead of a real Elasticsearch cluster.
95+
The `--mockserver` flag allows to run the benchmark against a "mock server", in this case Nginx, to understand a
96+
theoretical performance of the client, without the overhead of a real Elasticsearch cluster.
9397

9498
```
9599
ELASTICSEARCH_URL=http://server:8000 go run benchmarks.go --dataset=small --count=1_000_000 --flush=2MB --warmup=0 --mockserver
@@ -117,8 +121,12 @@ the size and structure of your data, the index settings and mappings, the cluste
117121
The benchmarks have been run in the following environment:
118122

119123
* OS: Ubuntu 18.04.4 LTS (5.0.0-1031-gcp)
120-
* Client: A `n2-standard-8` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (8 vCPUs/32GB RAM)
121-
* Server: A `n2-standard-16` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (16 vCPUs/64GB RAM)
122-
* Disk: A [local SSD](https://cloud.google.com/compute/docs/disks#localssds) formatted as `ext4` on NVMe interface for Elasticsearch data
123-
* A single-node Elasticsearch cluster, `7.6.0`, [default distribution](https://www.elastic.co/downloads/elasticsearch), installed from a TAR, with 4GB locked for heap
124+
* Client: A `n2-standard-8` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (8
125+
vCPUs/32GB RAM)
126+
* Server: A `n2-standard-16` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (16
127+
vCPUs/64GB RAM)
128+
* Disk: A [local SSD](https://cloud.google.com/compute/docs/disks#localssds) formatted as `ext4` on NVMe interface for
129+
Elasticsearch data
130+
* A single-node Elasticsearch cluster, `7.6.0`, [default distribution](https://www.elastic.co/downloads/elasticsearch),
131+
installed from a TAR, with 4GB locked for heap
124132
* Nginx 1.17.8 with [`nginx.conf`](etc/nginx.conf)

_examples/bulk/benchmarks/benchmarks.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
//go:build ignore
1819
// +build ignore
1920

2021
// This example demonstrates indexing documents using the esutil.BulkIndexer helper.
@@ -41,8 +42,8 @@ import (
4142
"github.com/mailru/easyjson"
4243
"github.com/valyala/fasthttp"
4344

45+
"github.com/elastic/elastic-transport-go/v8/elastictransport"
4446
"github.com/elastic/go-elasticsearch/v8"
45-
"github.com/elastic/go-elasticsearch/v8/estransport"
4647
"github.com/elastic/go-elasticsearch/v8/esutil"
4748

4849
"github.com/elastic/go-elasticsearch/v8/_examples/bulk/benchmarks/model"
@@ -108,7 +109,7 @@ func main() {
108109
}
109110

110111
if debug {
111-
clientCfg.Logger = &estransport.ColorLogger{Output: os.Stdout, EnableRequestBody: true, EnableResponseBody: true}
112+
clientCfg.Logger = &elastictransport.ColorLogger{Output: os.Stdout, EnableRequestBody: true, EnableResponseBody: true}
112113
}
113114

114115
es, _ := elasticsearch.NewClient(clientCfg)

_examples/bulk/benchmarks/data/httplog/mapping.json

-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
"_meta": {
44
"version": "8-SNAPSHOT"
55
},
6-
76
"dynamic_templates": [
87
{
98
"labels": {
@@ -60,9 +59,7 @@
6059
}
6160
}
6261
],
63-
6462
"date_detection": false,
65-
6663
"properties": {
6764
"@timestamp": {
6865
"type": "date"

_examples/bulk/benchmarks/data/small/mapping.json

+12-5
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,20 @@
33
"_meta": {
44
"version": "8-SNAPSHOT"
55
},
6-
76
"properties": {
8-
"count": { "type": "long" },
9-
"tags": { "type": "keyword" },
10-
"time": { "type": "date" },
11-
"title": { "type": "text" }
7+
"count": {
8+
"type": "long"
9+
},
10+
"tags": {
11+
"type": "keyword"
12+
},
13+
"time": {
14+
"type": "date"
15+
},
16+
"title": {
17+
"type": "text"
1218
}
1319
}
1420
}
1521
}
22+
}

_examples/bulk/benchmarks/go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ replace github.com/elastic/go-elasticsearch/v8 => ../../..
77
require (
88
github.com/dustin/go-humanize v1.0.0
99
github.com/elastic/go-elasticsearch/v8 v8.0.0-20210817150010-57d659deaca7
10+
github.com/fatih/color v1.13.0 // indirect
1011
github.com/mailru/easyjson v0.7.1
1112
github.com/montanaflynn/stats v0.6.3
12-
github.com/valyala/fasthttp v1.9.0
1313
)

_examples/bulk/benchmarks/go.sum

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
2+
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
3+
github.com/elastic/elastic-transport-go/v8 v8.0.0-20211202110751-50105067ef27 h1:O58SwZ7pdt7Lzy8JqpJubJlgDbr9jVV7ro4HFPvaZDw=
4+
github.com/elastic/elastic-transport-go/v8 v8.0.0-20211202110751-50105067ef27/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
5+
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
6+
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
7+
github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
8+
github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
9+
github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
10+
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
11+
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
12+
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
13+
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
14+
github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk=
15+
github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
16+
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
17+
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
18+
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
19+
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

_examples/bulk/default.go

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
//go:build ignore
1819
// +build ignore
1920

2021
// This example demonstrates indexing documents using the Elasticsearch "Bulk" API

_examples/bulk/go.mod

+5-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ go 1.11
55
replace github.com/elastic/go-elasticsearch/v8 => ../..
66

77
require (
8-
github.com/cenkalti/backoff/v4 v4.0.0
9-
github.com/dustin/go-humanize v1.0.0
10-
github.com/elastic/go-elasticsearch/v8 master
8+
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
9+
github.com/dustin/go-humanize v1.0.0 // indirect
10+
github.com/elastic/elastic-transport-go/v8 v8.0.0-20211202110751-50105067ef27 // indirect
11+
github.com/elastic/go-elasticsearch/v8 v8.0.0-20211123103400-5f8a17a2322f // indirect
12+
github.com/mailru/easyjson v0.7.7 // indirect
1113
)

_examples/bulk/go.sum

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo=
2+
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
3+
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
4+
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
5+
github.com/elastic/elastic-transport-go/v8 v8.0.0-20211202110751-50105067ef27 h1:O58SwZ7pdt7Lzy8JqpJubJlgDbr9jVV7ro4HFPvaZDw=
6+
github.com/elastic/elastic-transport-go/v8 v8.0.0-20211202110751-50105067ef27/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
7+
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
8+
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
9+
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
10+
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=

_examples/bulk/indexer.go

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
//go:build ignore
1819
// +build ignore
1920

2021
// This example demonstrates indexing documents using the esutil.BulkIndexer helper.

0 commit comments

Comments
 (0)