修改源码支持mysql查询

config/jdbc.yml

@@ -0,0 +1,14 @@
+## 新词mysql连接地址
+jdbc:
+  driver: com.mysql.cj.jdbc.Driver
+  url: jdbc:mysql://10.168.1.47:18080/smart-search?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai
+  username: root
+  password: Taiji@2023#data
+
+
+## redis 连接地址
+redis:
+  host: 10.168.1.47
+  port: 18084
+  timeout: 5000
+  password: Taiji@2024#data

pom.xml

@@ -83,6 +83,17 @@
             <scope>compile</scope>
         </dependency>
 
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.21</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson</artifactId>
+            <version>3.11.5</version>
+        </dependency>
 
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

src/main/assemblies/plugin.xml

@@ -41,5 +41,21 @@
                 <include>org.apache.httpcomponents:httpclient</include>
             </includes>
         </dependencySet>
+        <dependencySet>
+            <outputDirectory/>
+            <useProjectArtifact>true</useProjectArtifact>
+            <useTransitiveFiltering>true</useTransitiveFiltering>
+            <includes>
+                <include>mysql:mysql-connector-java</include>
+            </includes>
+        </dependencySet>
+        <dependencySet>
+            <outputDirectory/>
+            <useProjectArtifact>true</useProjectArtifact>
+            <useTransitiveFiltering>true</useTransitiveFiltering>
+            <includes>
+                <include>org.redisson:redisson</include>
+            </includes>
+        </dependencySet>
     </dependencySets>
 </assembly>

src/main/java/org/wltea/analyzer/dic/DatabaseMonitor.java

@@ -0,0 +1,294 @@
+package org.wltea.analyzer.dic;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.SpecialPermission;
+import org.redisson.api.RBucket;
+import org.redisson.api.RedissonClient;
+import org.wltea.analyzer.help.ESPluginLoggerFactory;
+import org.wltea.analyzer.util.RedisUtils;
+
+import java.net.InetAddress;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * 建立数据库监控线程
+ */
+public class DatabaseMonitor implements Runnable {
+
+    private static final Logger logger = ESPluginLoggerFactory.getLogger(DatabaseMonitor.class.getName());
+
+    public static final String PATH_JDBC_YML = "jdbc.yml";
+
+    private static final String JDBC_URL = "jdbc.url";
+    private static final String JDBC_USERNAME = "jdbc.username";
+    private static final String JDBC_PASSWORD = "jdbc.password";
+    private static final String JDBC_DRIVER = "jdbc.driver";
+
+    private static final String REDIS_HOST = "redis.host";
+    private static final String REDIS_PORT = "redis.port";
+    private static final String REDIS_TIMEOUT = "redis.timeout";
+    private static final String REDIS_PASSWORD = "redis.password";
+
+    private Properties propsRedis = new Properties();
+
+
+    public String getUrl() {
+        return Dictionary.getSingleton().getProperty(JDBC_URL);
+    }
+
+    public String getUsername() {
+        return Dictionary.getSingleton().getProperty(JDBC_USERNAME);
+    }
+
+    public String getPassword() {
+        return Dictionary.getSingleton().getProperty(JDBC_PASSWORD);
+    }
+
+    public String getDriver() {
+        return Dictionary.getSingleton().getProperty(JDBC_DRIVER);
+    }
+
+    /**
+     * 加载MySQL驱动
+     */
+    public DatabaseMonitor() {
+
+        SpecialPermission.check();
+
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+
+            try {
+
+                Class.forName(getDriver());
+
+                propsRedis.put(REDIS_HOST, Dictionary.getSingleton().getProperty(REDIS_HOST));
+                propsRedis.put(REDIS_PORT, Dictionary.getSingleton().getProperty(REDIS_PORT));
+                propsRedis.put(REDIS_TIMEOUT, Dictionary.getSingleton().getProperty(REDIS_TIMEOUT));
+                propsRedis.put(REDIS_PASSWORD, Dictionary.getSingleton().getProperty(REDIS_PASSWORD));
+
+            } catch (ClassNotFoundException e) {
+
+                logger.error("mysql jdbc driver not found", e);
+            }
+            return null;
+        });
+
+
+    }
+
+    @Override
+    public void run() {
+
+        SpecialPermission.check();
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+
+            //从mysql加载热更新词典
+            loadExtDictByMysql();
+
+            return null;
+        });
+
+    }
+
+    /**
+     * 从mysql加载热更新词典
+     */
+    private void loadExtDictByMysql() {
+
+        Connection conn = null;
+        Statement stmt = null;
+        ResultSet rs = null;
+
+        try {
+
+            Long thisSynonymVersion = getRedisSynonymVersion();
+
+            conn = DriverManager.getConnection(getUrl(),getUsername(),getPassword());
+
+            stmt = conn.createStatement();
+
+            String sql = "select *,UNIX_TIMESTAMP(update_time) as version from sys_hot_word where UNIX_TIMESTAMP(update_time) >= "+thisSynonymVersion;
+
+            rs = stmt.executeQuery(sql);
+
+            List<String> stopWordList = new ArrayList<>();
+            List<String> mainWordList = new ArrayList<>();
+
+            List<String> disStopWordList = new ArrayList<>();
+            List<String> disMainWordList = new ArrayList<>();
+
+            while (rs.next()) {
+
+                String theWord = rs.getString("word");
+                String status = rs.getString("status");
+                String stopWord = rs.getString("stop_word");
+                Long version = rs.getLong("version");
+
+                if(version > thisSynonymVersion){
+
+                    thisSynonymVersion = version;
+
+                }
+
+                if("1".equals(status)){
+
+                    if("1".equals(stopWord)){
+
+                        stopWordList.add(theWord);
+
+                    }else{
+                        mainWordList.add(theWord);
+                    }
+
+                }else{
+
+                    if("1".equals(stopWord)){
+
+                        disStopWordList.add(theWord);
+
+                    }else{
+                        disMainWordList.add(theWord);
+                    }
+
+                }
+            }
+
+            setRedisSynonymVersion(thisSynonymVersion);
+
+            if(stopWordList.size() > 0){
+
+                Dictionary.getSingleton().addStopWords(stopWordList);
+
+            }
+
+            if(mainWordList.size() > 0){
+                Dictionary.getSingleton().addMainWords(mainWordList);
+            }
+
+            if(disStopWordList.size() > 0){
+                Dictionary.getSingleton().disableStopWords(disStopWordList);
+            }
+
+            if(disMainWordList.size() > 0){
+                Dictionary.getSingleton().disableMainWords(disMainWordList);
+            }
+
+            logger.info("从mysql加载热更新词典成功!");
+
+        } catch (Exception e) {
+
+            logger.error("从mysql加载热更新词典失败,错误信息为:", e);
+
+        } finally {
+            if (rs != null) {
+                try {
+                    rs.close();
+                } catch (SQLException e) {
+                    logger.error("error", e);
+                }
+            }
+            if (stmt != null) {
+                try {
+                    stmt.close();
+                } catch (SQLException e) {
+                    logger.error("error", e);
+                }
+            }
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (SQLException e) {
+                    logger.error("error", e);
+                }
+            }
+        }
+    }
+
+    /**
+     * 获取redis中同义词版本号信息
+     * 用于判断同义词是否需要进行重新加载
+     * @return
+     */
+    public Long getRedisSynonymVersion() {
+
+        Long thisSynonymVersion = 0L;
+
+        RedissonClient redissonClient = RedisUtils.getRedissonClient(propsRedis);
+
+        try{
+
+            String ip = InetAddress.getLocalHost().getHostAddress();
+
+            RBucket<String> ipRBucket =  redissonClient.getBucket("es_version_"+ip);
+
+            if(ipRBucket.isExists()){
+
+                String version = ipRBucket.get();
+
+                if(version !=null && !"".equals(version)){
+
+                    thisSynonymVersion = Long.parseLong(version);
+
+                }
+
+
+
+            }
+
+        }catch (Exception exception){
+
+            exception.printStackTrace();
+
+            logger.error("获取redis中同义词版本号信息失败！"+exception);
+
+        }finally {
+
+            if(redissonClient !=null && !redissonClient.isShutdown()){
+
+                redissonClient.shutdown();
+            }
+        }
+
+        return thisSynonymVersion;
+
+    }
+
+    /**
+     * 设置redis中同义词版本号信息
+     * 用于判断同义词是否需要进行重新加载
+     * @return
+     */
+    public void setRedisSynonymVersion(Long setSynonymVersion) {
+
+
+        RedissonClient redissonClient = RedisUtils.getRedissonClient(propsRedis);
+
+        try{
+
+            String ip = InetAddress.getLocalHost().getHostAddress();
+
+            RBucket<String> ipRBucket =  redissonClient.getBucket("es_version_"+ip);
+
+            ipRBucket.set(String.valueOf(setSynonymVersion));
+
+        }catch (Exception exception){
+
+            exception.printStackTrace();
+
+            logger.error("设置redis中同义词版本号信息失败！"+exception);
+
+        }finally {
+
+            if(redissonClient !=null && !redissonClient.isShutdown()){
+
+                redissonClient.shutdown();
+            }
+        }
+
+    }
+}

src/main/java/org/wltea/analyzer/dic/Dictionary.java

@@ -130,7 +130,7 @@ public class Dictionary {
 		}
 	}
 
-	private String getProperty(String key){
+	public String getProperty(String key){
 		if(props!=null){
 			return props.getProperty(key);
 		}
@@ -164,6 +164,8 @@ public class Dictionary {
 						for (String location : singleton.getRemoteExtStopWordDictionarys()) {
 							pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
 						}
+						//建立数据库监控线程
+						pool.scheduleAtFixedRate(new DatabaseMonitor(),10,120,TimeUnit.SECONDS);
 					}
 
 				}
@@ -299,6 +301,68 @@ public class Dictionary {
 		return singleton;
 	}
 
+	/**
+	 * 批量加载新停顿词条
+	 *
+	 * @param words
+	 *            Collection<String>词条列表
+	 */
+	public void addStopWords(Collection<String> words) {
+		if (words != null) {
+			for (String word : words) {
+				if (word != null) {
+					// 批量加载词条到主内存词典中
+					singleton._StopWords.fillSegment(word.trim().toCharArray());
+				}
+			}
+		}
+	}
+
+	/**
+	 * 批量移除（屏蔽）停顿词条
+	 */
+	public void disableStopWords(Collection<String> words) {
+		if (words != null) {
+			for (String word : words) {
+				if (word != null) {
+					// 批量屏蔽词条
+					singleton._StopWords.disableSegment(word.trim().toCharArray());
+				}
+			}
+		}
+	}
+
+	/**
+	 * 批量加载新词条
+	 *
+	 * @param words
+	 *            Collection<String>词条列表
+	 */
+	public void addMainWords(Collection<String> words) {
+		if (words != null) {
+			for (String word : words) {
+				if (word != null) {
+					// 批量加载词条到主内存词典中
+					singleton._MainDict.fillSegment(word.trim().toCharArray());
+				}
+			}
+		}
+	}
+
+	/**
+	 * 批量移除（屏蔽）词条
+	 */
+	public void disableMainWords(Collection<String> words) {
+		if (words != null) {
+			for (String word : words) {
+				if (word != null) {
+					// 批量屏蔽词条
+					singleton._MainDict.disableSegment(word.trim().toCharArray());
+				}
+			}
+		}
+	}
+
 
 	/**
 	 * 批量加载新词条

src/main/java/org/wltea/analyzer/util/RedisUtils.java

@@ -0,0 +1,40 @@
+package org.wltea.analyzer.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.redisson.Redisson;
+import org.redisson.api.RedissonClient;
+import org.redisson.config.Config;
+import org.wltea.analyzer.dic.DatabaseMonitor;
+import org.wltea.analyzer.help.ESPluginLoggerFactory;
+
+import java.util.Properties;
+import java.util.Set;
+
+public class RedisUtils {
+
+    private static final Logger logger = ESPluginLoggerFactory.getLogger(RedisUtils.class.getName());
+
+    private RedisUtils(){
+
+    }
+
+    /**
+     * 定义静态方法，返回RedissonClient对象
+     */
+    public static RedissonClient getRedissonClient(Properties props){
+
+        //1、创建redission的config对象并配置redis服务器
+        Config config = new Config();
+        config.useSingleServer().setAddress("redis://"+props.getProperty("redis.host")+":"+props.getProperty("redis.port"))
+        .setConnectTimeout(Integer.parseInt(props.getProperty("redis.timeout"))).setPassword(props.getProperty("redis.password"));
+
+        //2、创建Redisson客户端
+        RedissonClient redisson = Redisson.create(config);
+
+        return redisson;
+    }
+
+
+
+}

src/main/resources/plugin-security.policy

@@ -1,4 +1,5 @@
 grant {
   // needed because of the hot reload functionality
   permission java.net.SocketPermission "*", "connect,resolve";
+  permission java.lang.RuntimePermission "setContextClassLoader";
 };