[insert]es海康抓拍图片转储S3并更新es图片地址

src/main/java/cn/com/taiji/esshipservice/listener/hlx/fusion/EsHkImagesDumpS3.java

@@ -3,6 +3,7 @@ package cn.com.taiji.esshipservice.listener.hlx.fusion;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import cn.com.taiji.esshipservice.utils.HttpsDownloadUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.config.RequestConfig;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -39,6 +40,7 @@ import static cn.com.taiji.esshipservice.constant.ShipHisTrackIndexConstants.*;
  * @date 2023/2/24
  */
 @Component
+@Slf4j
 public class EsHkImagesDumpS3 {
 
     @Autowired
@@ -52,7 +54,7 @@ public class EsHkImagesDumpS3 {
     static {
         RequestConfig requestConfig = RequestConfig.custom()
                 .setConnectTimeout(5000)
-                .setSocketTimeout(60000)
+                .setSocketTimeout(6000)
                 .build();
         RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
         builder.setRequestConfig(requestConfig).setHttpAsyncResponseConsumerFactory(
@@ -62,28 +64,24 @@ public class EsHkImagesDumpS3 {
         COMMON_OPTIONS = builder.build();
     }
 
-    ExecutorService threadPool = new ThreadPoolExecutor( 15,35,10L, TimeUnit.SECONDS, new LinkedBlockingDeque<>(10), Executors.defaultThreadFactory(), new ThreadPoolExecutor.DiscardPolicy());
+//    ExecutorService threadPool = new ThreadPoolExecutor( 15,35,10L, TimeUnit.SECONDS, new LinkedBlockingDeque<>(10), Executors.defaultThreadFactory(), new ThreadPoolExecutor.DiscardPolicy());
 
-    @Scheduled(cron = "0 0 */3 * * ?")
-    public void scheduled(){
+    @Scheduled(fixedRate=60*60*1000)
+    public void scheduled() throws IOException {
         Calendar calendar = Calendar.getInstance();
-        calendar.set(Calendar.HOUR_OF_DAY, calendar.get(Calendar.HOUR_OF_DAY) -6);
+        calendar.set(Calendar.HOUR_OF_DAY, calendar.get(Calendar.HOUR_OF_DAY) -2);
         Date time1 = calendar.getTime();
         Calendar calendar2 = Calendar.getInstance();
-        calendar2.set(Calendar.HOUR_OF_DAY, calendar2.get(Calendar.HOUR_OF_DAY) -3);
+        calendar2.set(Calendar.HOUR_OF_DAY, calendar2.get(Calendar.HOUR_OF_DAY) -1);
         Date time2 = calendar2.getTime();
         SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
-        SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
         String startTime = sdf1.format(time1);
         String endTime = sdf1.format(time2);
-        String startTimeShip = sdf2.format(time1);
-        String endTimeShip = sdf2.format(time2);
-        imagesDumpS3Ship(startTimeShip,endTimeShip);
         imagesDumpS3Car(startTime,endTime);
         imagesDumpS3Face(startTime,endTime);
     }
 
-    public void imagesDumpS3Car(String startTime,String endTime){
+    public void imagesDumpS3Car(String startTime,String endTime) throws IOException {
         //设置查询索引
         SearchRequest searchRequest = new SearchRequest(INDEX_SEAT_HIK_CAPTURE_CAR_TRACKS);
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@@ -92,10 +90,10 @@ public class EsHkImagesDumpS3 {
         searchSourceBuilder.query(boolQueryBuilder);
         searchSourceBuilder
                 .trackTotalHits(true)
-                .size(1000000)
-                .timeout(TimeValue.timeValueHours(1L))
-                .timeout(TimeValue.timeValueMinutes(30L))
-                .timeout(TimeValue.timeValueSeconds(500L));
+                .size(1000000);
+//                .timeout(TimeValue.timeValueHours(1L))
+//                .timeout(TimeValue.timeValueMinutes(30L))
+//                .timeout(TimeValue.timeValueSeconds(500L));
         searchRequest.source(searchSourceBuilder);
         SearchResponse search = null;
         try {
@@ -145,8 +143,9 @@ public class EsHkImagesDumpS3 {
                 s3targetPicUrl = httpsDownloadUtils.downloadFileHttps(targetPicUrl.toString());
                 maps.put("targetPicUrl", s3targetPicUrl);
 //                }
+
             }
-            if (null != s3targetPicUrl || null != s3plateImagePath) {
+            if (!maps.isEmpty()) {
                 bulkRequest.add(new UpdateRequest(INDEX_SEAT_HIK_CAPTURE_CAR_TRACKS, hit.getId()).doc(maps));
                 num++;
             }
@@ -169,7 +168,7 @@ public class EsHkImagesDumpS3 {
         }
     }
 
-    public void imagesDumpS3Face(String startTime,String endTime){
+    public void imagesDumpS3Face(String startTime,String endTime) throws IOException {
         //设置查询索引
         SearchRequest searchRequest = new SearchRequest(INDEX_SEAT_HIK_CAPTURE_FACE_TRACKS);
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@@ -178,10 +177,10 @@ public class EsHkImagesDumpS3 {
         searchSourceBuilder.query(boolQueryBuilder);
         searchSourceBuilder
                 .trackTotalHits(true)
-                .size(1000000)
-                .timeout(TimeValue.timeValueHours(1L))
-                .timeout(TimeValue.timeValueMinutes(30L))
-                .timeout(TimeValue.timeValueSeconds(500L));
+                .size(1000000);
+//                .timeout(TimeValue.timeValueHours(1L))
+//                .timeout(TimeValue.timeValueMinutes(30L))
+//                .timeout(TimeValue.timeValueSeconds(500L));
         searchRequest.source(searchSourceBuilder);
         SearchResponse search = null;
         try {
@@ -232,7 +231,7 @@ public class EsHkImagesDumpS3 {
                 maps.put("facePicUrl", s3facePicUrl);
 //                }
             }
-            if (null != s3facePicUrl || null != s3bkgUrl) {
+            if (!maps.isEmpty()) {
                 bulkRequest.add(new UpdateRequest(INDEX_SEAT_HIK_CAPTURE_FACE_TRACKS, hit.getId()).doc(maps));
                 num++;
             }
@@ -255,78 +254,5 @@ public class EsHkImagesDumpS3 {
         }
     }
 
-
-    public void imagesDumpS3Ship(String startTime,String endTime){
-        //设置查询索引
-        SearchRequest searchRequest = new SearchRequest(INDEX_SEAT_HIK_CAPTURE_SHIP_TRACKS);
-        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-        BoolQueryBuilder boolQueryBuilder= QueryBuilders.boolQuery();
-        boolQueryBuilder.filter(QueryBuilders.rangeQuery("captureTime").gte(startTime).lte(endTime));
-        searchSourceBuilder.query(boolQueryBuilder);
-        searchSourceBuilder
-                .trackTotalHits(true)
-                .size(1000000)
-                .timeout(TimeValue.timeValueHours(1L))
-                .timeout(TimeValue.timeValueMinutes(30L))
-                .timeout(TimeValue.timeValueSeconds(500L));
-        searchRequest.source(searchSourceBuilder);
-        SearchResponse search = null;
-        try {
-            search = client.search(searchRequest, COMMON_OPTIONS);
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        SearchHits searchHits = search.getHits();
-        SearchHit[] hits = searchHits.getHits();
-        BulkRequest bulkRequest = new BulkRequest();
-        Integer num = 0;
-        for (SearchHit hit : hits) {
-            Map<String, Object> map = hit.getSourceAsMap();
-            Map<String, Object> maps = new HashMap<>();
-            Object targetImageUrl = map.get("targetImageUrl");
-            String s3targetImageUrl = null;
-            if (null != targetImageUrl && !targetImageUrl.equals("")){
-                s3targetImageUrl = httpsDownloadUtils.downloadFileHttps(targetImageUrl.toString());
-                maps.put("targetImageUrl",s3targetImageUrl);
-            }
-            Object targetImages1 = map.get("targetImages");
-            Integer text = 0;
-            if (null != targetImages1  && !targetImages1.equals("")){
-                List<String> targetImages = JSONArray.parseArray(targetImages1.toString(),String.class);
-                List<String> newTargetImages = new CopyOnWriteArrayList<>();
-                targetImages.forEach(string->{
-                    String s3targetImages = httpsDownloadUtils.downloadFileHttps(string);
-                    if (null != s3targetImages && !s3targetImages.equals("")){
-                        newTargetImages.add(s3targetImages);
-                    }
-                });
-                if (null != newTargetImages){
-                    maps.put("targetImages",JSON.toJSONString(newTargetImages));
-                    text = 1;
-                }
-            }
-            if (null != s3targetImageUrl || text!=0){
-                bulkRequest.add(new UpdateRequest(INDEX_SEAT_HIK_CAPTURE_SHIP_TRACKS, hit.getId()).doc(maps));
-                num++;
-            }
-            if (num == 100){
-                try {
-                    BulkResponse bulk = client.bulk(bulkRequest, RequestOptions.DEFAULT);
-                    num = 0;
-                    bulkRequest.requests().clear();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-        try {
-            if (bulkRequest.requests().size()>0){
-                BulkResponse bulk = client.bulk(bulkRequest, RequestOptions.DEFAULT);
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-
-    }
 }
 

src/main/resources/application-prod.yml

@@ -58,12 +58,12 @@ spring:
 
 taiji:
   openservice:
-    enable: false
-    url: http://127.0.0.1:8080
+    enable: true
+    url: http://74.10.28.87:9002
   elasticsearch.rest:
-    uris: 120.25.233.10:9200
-    username: elastic       #如果你设置了基于x-pack的验证就要填写账号和密码
-    password: root@2022!       #没有则不用配置
+    uris: 74.10.28.65:9200,74.10.28.66:9200,74.10.28.67:9200,74.10.28.68:9200,74.10.28.69:9200
+    username: ax_seat       #如果你设置了基于x-pack的验证就要填写账号和密码
+    password: ax_seat       #没有则不用配置
     connection-timeout: 100 #连接超时
     max-connection: 100  #最大连接数
   hbase.zookeeper:
@@ -338,7 +338,7 @@ file:
     access-key: ax_seat_acc
     secret-key: ax_seat_AxSeatPW
     bucket: ax_es_hik_images
-    prefix: seat/file/resource
+    prefix: ax_es_hik_images
     domain: http://74.10.28.62:81/
     ip: 74.10.28.62
     port: 81

target/classes/application-prod.yml

@@ -58,12 +58,12 @@ spring:
 
 taiji:
   openservice:
-    enable: false
-    url: http://127.0.0.1:8080
+    enable: true
+    url: http://74.10.28.87:9002
   elasticsearch.rest:
-    uris: 120.25.233.10:9200
-    username: elastic       #如果你设置了基于x-pack的验证就要填写账号和密码
-    password: root@2022!       #没有则不用配置
+    uris: 74.10.28.65:9200,74.10.28.66:9200,74.10.28.67:9200,74.10.28.68:9200,74.10.28.69:9200
+    username: ax_seat       #如果你设置了基于x-pack的验证就要填写账号和密码
+    password: ax_seat       #没有则不用配置
     connection-timeout: 100 #连接超时
     max-connection: 100  #最大连接数
   hbase.zookeeper:
@@ -338,7 +338,7 @@ file:
     access-key: ax_seat_acc
     secret-key: ax_seat_AxSeatPW
     bucket: ax_es_hik_images
-    prefix: seat/file/resource
+    prefix: ax_es_hik_images
     domain: http://74.10.28.62:81/
     ip: 74.10.28.62
     port: 81