golang ES 聚合查询

Es通过脚本的方式计算每条数据
go demo

	fsq := elastic.NewFunctionScoreQuery().BoostMode("replace")
	script := `if(doc['device_count'].size() != 0 && doc['device_count'].value != 0 )
							{
							double price = 0.000001;
								if (doc['discount_price'].size() != 0 && doc['discount_price'].value != 0)
								{
									price = doc['discount_price'].value;
								}
							return doc['device_count'].value * price
							}`

	fsq = fsq.AddScoreFunc(elastic.NewScriptFunction(elastic.NewScript(script)))
	
	boolQuery := elastic.NewBoolQuery()
	boolQuery.Filter(elastic.NewTermsQuery("city.code", "130200"))
	
	search := elastic.NewSearchService(es.EsClient).Index(esMediaResourceIndex).Size(10)

	search.Query(fsq.Query(boolQuery))
	searchResult, err := search.
		Pretty(true).
		Do(context.Background())

	if err != nil {
		panic(err)
	}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

求最大值，最小，平均，求和

	boolQuery := elastic.NewBoolQuery()
	boolQuery.Filter(elastic.NewTermsQuery("city.code", "130200"))

	deviceSumAggs := elastic.NewSumAggregation().Field("device_count")
	deviceAvgAggs := elastic.NewAvgAggregation().Field("device_count")
	deviceMinAggs := elastic.NewMinAggregation().Field("device_count")
	deviceMaxAggs := elastic.NewMaxAggregation().Field("device_count")

	search := elastic.NewSearchService(es.EsClient).Index(esMediaResourceIndex).Size(10)
	search.Query(boolQuery)
	searchResult, err := search.
		Aggregation("device_sum", deviceSumAggs).
		Aggregation("device_avg", deviceAvgAggs).
		Aggregation("device_min", deviceMinAggs).
		Aggregation("device_max", deviceMaxAggs).
		Pretty(true).
		Do(context.Background())

	if err != nil {
		panic(err)
	}

	if deviceSumAgg, found := searchResult.Aggregations.Sum("device_sum"); found {
	    if deviceSumAgg.Value != nil {
	    	fmt.Println(*deviceSumAgg.Value)
		}
	}
	if deviceAvgAgg, found := searchResult.Aggregations.Sum("device_avg"); found {
		if deviceAvgAgg.Value != nil {
			fmt.Println(*deviceAvgAgg.Value)
		}
	}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

es分桶统计计数
demo 统计不同媒体类型的设备数

	boolQuery := elastic.NewBoolQuery()
	boolQuery.Filter(elastic.NewTermsQuery("city.code", "130200"))

	mediaCodeTermAggs := elastic.NewTermsAggregation().Field("media_type.code")
	mediaCodeTermAggs.SubAggregation("device_sum", elastic.NewSumAggregation().Field("device_count"))

	search := elastic.NewSearchService(es.EsClient).Index(esMediaResourceIndex).Size(10)
	search.Query(boolQuery)
	searchResult, err := search.
		Aggregation("media_type_sum", mediaCodeTermAggs).
		Pretty(true).
		Do(context.Background())

	if err != nil {
		panic(err)
	}

	if mediaCodeTermAgg, found := searchResult.Aggregations.Terms("media_type_sum"); found {
	    for _, buckt := range mediaCodeTermAgg.Buckets {
			fmt.Println(buckt.Key)
	    	if agg, ok := buckt.Aggregations.Sum("device_sum"); ok{
	    		fmt.Println(*agg.Value)
			}
		}
	}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

es 围栏的方式查询数据

package main

import (
	"context"
	"fmt"
	"github.com/olivere/elastic/v7"
	"wtp-media-house/internal/app/es"
	"wtp-media-house/internal/app/media_resource/domain/params"
	"wtp-media-house/internal/pkg/logger"
)

var (
	esIndex = "unidata_heat_with_tag_202201"
	esMediaResourceIndex = "wtp_media_resource"
)




func main() {

	genoInfos := []params.GeoFence {
		{
			Name:         "",
			GeofenceType: 1,
			Lon:          116.424268,
			Lat:          39.978495,
			Radius:       200,
			Polygon:      nil,
		},
	}

	boolQuery := elastic.NewBoolQuery()
	subBoolQuery := elastic.NewBoolQuery()
	for _, geoInfo := range genoInfos {
		query := GeofenceUniDataQuery(geoInfo)
		if query != nil {
			subBoolQuery.Should(query)
		}
	}
	boolQuery.Filter(subBoolQuery)

	search := es.EsClient.Search().Index(esMediaResourceIndex)
	search.Size(10)
	search.Query(boolQuery)

	searchResult, err := search.
		Pretty(true).
		Do(context.Background())

	if err != nil {
		panic(err)
	}

	logger.WayzLog.Info(fmt.Sprintf("【Ta特征查询】 Query took %d milliseconds ", searchResult.TookInMillis))

}

func GeofenceUniDataQuery(in interface{}) elastic.Query {
	fence, ok := in.(params.GeoFence)
	if !ok {
		return nil
	}
	fenceType := fence.GeofenceType

	//圆形
	if fenceType == 1 && fence.Lon != 0 && fence.Lat != 0 {
		return elastic.NewGeoDistanceQuery("location").
			Lat(fence.Lat).
			Lon(fence.Lon).
			Distance(fmt.Sprintf("%fm", fence.Radius))
	}

	//多边形
	if fenceType == 2 && len(fence.Polygon) != 0 {
		geoQuery := elastic.NewGeoPolygonQuery("location")
		for _, polygon := range fence.Polygon {
			if len(polygon) == 2 {
				geoQuery.AddPoint(polygon[1], polygon[0])
			}
		}
		return geoQuery
	}

	//矩形（视窗）
	if fenceType == 3 && len(fence.Polygon) == 2 {
		point1 := fence.Polygon[0]
		point2 := fence.Polygon[1]
		if len(point1) == 2 && len(point2) == 2 {
			geoQuery := elastic.NewGeoBoundingBoxQuery("location").
				TopLeft(max(point1[1], point2[1]), min(point1[0], point2[0])).
				BottomRight(min(point1[1], point2[1]), max(point1[0], point2[0]))
			return geoQuery
		}
	}
	return nil
}

func max(a, b float64) float64 {
	if a < b {
		return b
	}
	return a
}

func min(a, b float64) float64 {
	if a > b {
		return b
	}
	return a
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

es 根据输入查询的参数分桶计算
demo 查询2个城市下的信息和uv

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/olivere/elastic/v7"
	"wtp-media-house/internal/app/es"
	"wtp-media-house/internal/pkg/config"
)

func main() {
	cityCodes := []string{
		"510100", "130200",
	}

	fetch := elastic.NewFetchSourceContext(true).Include("city") // 只取部分数据

	aggs := elastic.NewFiltersAggregation()

	for _, city := range cityCodes{
		boolQuery := elastic.NewBoolQuery()
		boolQuery.Filter(elastic.NewTermsQuery("city.code", city))
		aggs.FilterWithName(city, boolQuery)
	}
	// 每个桶取2条数
	aggs.SubAggregation("city_info", elastic.NewTopHitsAggregation().FetchSourceContext(fetch).Size(2))
	// 统计每个桶的uv
	aggs.SubAggregation("city_uv", elastic.NewSumAggregation().Field("resident.uv"))
	
	search := elastic.NewSearchService(es.EsHeatClient).Index(config.ES_UNIDATA_HEAR_WITH_TAG)
	searchResult, err := search.
		Aggregation("city_code", aggs).
		Pretty(true).
		Do(context.Background())

	if err != nil {
		panic(err)
	}

	type CityInfo struct {
		City struct {
			Name string `json:"name"`
			Code string `json:"code"`
		} `json:"city"`
	}
	resultCityInfo := make(map[string][]string)
	resultCityUv := make(map[string]int)

	if agg, found := searchResult.Aggregations.Filters("city_code"); found {
		for key, bucket :=range agg.NamedBuckets {
			resultCityInfo[key] = []string{}
			if city, ok := bucket.Aggregations.TopHits("city_info"); ok {
				if city.Hits.TotalHits.Value > 0 {
				    for _, hit := range city.Hits.Hits {
				    	var cityName CityInfo
				    	json.Unmarshal(hit.Source, &cityName)
						resultCityInfo[key]= append(resultCityInfo[key], cityName.City.Name)
					}
				}
			}
			if cityUv, ok := bucket.Aggregations.Sum("city_uv"); ok {
				if cityUv.Value != nil {
					resultCityUv[key] = int(*cityUv.Value)
				}
			}
		}
	}
	fmt.Println(resultCityInfo)
	fmt.Println(resultCityUv)
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72