限流和反爬虫机制

config-server/src/main/resources/shared/gateway-service-local.yml

@@ -10,9 +10,18 @@ ratelimiter-conf:
 spring:
   #redis
   redis:
+    database:
+      cache: 0 # Redis限流缓存索引
     host: 192.168.2.236  #Redis服务器地址
     port: 6378 # Redis服务器连接端口(本地环境端口6378，其他环境端口是6379)
     password: lantone # Redis服务器连接密码（默认为空）
+    lettuce:
+      pool:
+        max-active: 8 # 连接池最大连接数（使用负值表示没有限制）
+        max-idle: 5 # 连接池中的最大空闲连接
+        max-wait: -1 # 连接池最大阻塞等待时间（使用负值表示没有限制）
+        min-idle: 0 # 连接池中的最小空闲连接
+    timeout: 20000 # 连接超时时间（毫秒）
   #mq
   rabbitmq:
     host: localhost

gateway-service/pom.xml

@@ -83,6 +83,11 @@
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-starter-stream-rabbit</artifactId>
         </dependency>
+        <!-- 必须加上，jedis依赖此  -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-pool2</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.diagbot</groupId>
             <artifactId>common</artifactId>

gateway-service/src/main/java/com/diagbot/config/RedisConfigurer.java

@@ -0,0 +1,169 @@
+package com.diagbot.config;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.annotation.CachingConfigurerSupport;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.cache.interceptor.KeyGenerator;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.data.redis.cache.RedisCacheConfiguration;
+import org.springframework.data.redis.cache.RedisCacheManager;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.connection.lettuce.LettucePoolingClientConfiguration;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.RedisSerializationContext;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+
+import java.time.Duration;
+
+@Configuration
+@EnableCaching
+@Slf4j
+public class RedisConfigurer extends CachingConfigurerSupport {
+
+    @Value("${spring.redis.database.cache}")
+    private String databaseCache;
+    @Value("${spring.redis.host}")
+    private String host;
+    @Value("${spring.redis.password}")
+    private String password;
+    @Value("${spring.redis.port}")
+    private int port;
+    @Value("${spring.redis.timeout}")
+    private int timeout;
+    @Value("${spring.redis.lettuce.pool.max-active}")
+    private int maxActive;
+    @Value("${spring.redis.lettuce.pool.max-idle}")
+    private int maxIdle;
+    @Value("${spring.redis.lettuce.pool.max-wait}")
+    private long maxWaitMillis;
+    @Value("${spring.redis.lettuce.pool.min-idle}")
+    private int minIdle;
+
+    @Autowired
+    @Qualifier("factoryForCache")
+    private LettuceConnectionFactory lettuceConnectionFactory;
+
+    @Bean
+    public GenericObjectPoolConfig getRedisConfig() {
+        GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
+        poolConfig.setMaxTotal(maxActive);
+        poolConfig.setMaxIdle(maxIdle);
+        poolConfig.setMaxWaitMillis(maxWaitMillis);
+        poolConfig.setMinIdle(minIdle);
+        return poolConfig;
+    }
+
+    @Bean
+    @Override
+    public CacheManager cacheManager() {
+        RedisCacheConfiguration redisCacheConfiguration = RedisCacheConfiguration.defaultCacheConfig()
+                // 设置 key为string序列化
+                .serializeKeysWith(RedisSerializationContext.SerializationPair.fromSerializer(new StringRedisSerializer()))
+                // 设置value为json序列化
+                .serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(getSerializer()))
+                // 不缓存空值
+                .disableCachingNullValues();
+        RedisCacheManager cacheManager = RedisCacheManager.builder(lettuceConnectionFactory)
+                .cacheDefaults(redisCacheConfiguration)
+                .transactionAware()
+                .build();
+        cacheManager.afterPropertiesSet();
+        log.info("RedisCacheManager config success");
+        return cacheManager;
+    }
+
+    @Bean(name = "springSessionDefaultRedisSerializer")
+    public GenericJackson2JsonRedisSerializer getGenericJackson2JsonRedisSerializer() {
+        return new GenericJackson2JsonRedisSerializer();
+    }
+
+    /**
+     * 缓存使用的redis
+     *
+     * @return
+     */
+    @Bean("factoryForCache")
+    @Primary
+    public LettuceConnectionFactory redisConnectionFactory() {
+        return getRedisConnectionFactory(Integer.valueOf(databaseCache));
+    }
+
+    @Bean(name = "redisTemplateForCache")
+    public RedisTemplate<String, Object> redisTemplate() {
+        return getRedisTemplate(lettuceConnectionFactory);
+    }
+
+    private Jackson2JsonRedisSerializer getSerializer() {
+        Jackson2JsonRedisSerializer jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<>(Object.class);
+        ObjectMapper om = new ObjectMapper();
+        om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
+        om.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL);
+        jackson2JsonRedisSerializer.setObjectMapper(om);
+        return jackson2JsonRedisSerializer;
+    }
+
+    @Bean
+    @Override
+    public KeyGenerator keyGenerator() {
+        //  设置自动key的生成规则，配置spring boot的注解，进行方法级别的缓存
+        // 使用：进行分割，可以很多显示出层级关系
+        // 这里其实就是new了一个KeyGenerator对象
+        return (target, method, params) -> {
+            StringBuilder sb = new StringBuilder();
+            sb.append(target.getClass().getName());
+            sb.append(":");
+            sb.append(method.getName());
+            for (Object obj : params) {
+                sb.append(":" + String.valueOf(obj));
+            }
+            String rsToUse = String.valueOf(sb);
+            log.info("自动生成Redis Key -> [{}]", rsToUse);
+            return rsToUse;
+        };
+    }
+
+    private LettuceConnectionFactory getRedisConnectionFactory(Integer database) {
+        RedisStandaloneConfiguration connection = new RedisStandaloneConfiguration();
+        connection.setHostName(host);
+        connection.setPort(port);
+        connection.setPassword(password);
+        connection.setDatabase(database);
+        GenericObjectPoolConfig poolConfig = getRedisConfig();
+        LettuceClientConfiguration builder = LettucePoolingClientConfiguration.builder()
+                .commandTimeout(Duration.ofMillis(timeout))
+                .poolConfig(poolConfig)
+                .shutdownTimeout(Duration.ZERO)
+                .build();
+        LettuceConnectionFactory factory = new LettuceConnectionFactory(connection, builder);
+        return factory;
+    }
+
+    private RedisTemplate<String, Object> getRedisTemplate(LettuceConnectionFactory factory) {
+        RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(factory);
+
+        // value值的序列化
+        redisTemplate.setValueSerializer(getSerializer());
+        redisTemplate.setHashValueSerializer(getSerializer());
+        // key的序列化采用StringRedisSerializer
+        redisTemplate.setKeySerializer(new StringRedisSerializer());
+        redisTemplate.setHashKeySerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+        return redisTemplate;
+    }
+}
+ 

gateway-service/src/main/java/com/diagbot/config/redislimiter/SysKeyResolver.java

@@ -33,7 +33,7 @@ public class SysKeyResolver implements KeyResolver {
             }
         }
 
-        String myKeyResolver = exchange.getRequest().getRemoteAddress().getHostName()
+        String myKeyResolver = exchange.getRequest().getRemoteAddress().getAddress().getHostAddress()
                 + "_" + fullUrl;
         ServerHttpRequest.Builder builder = exchange.getRequest().mutate();
         builder.header("myKeyResolver", myKeyResolver);

gateway-service/src/main/java/com/diagbot/config/redislimiter/SystemRedisRateLimiter.java

@@ -1,12 +1,16 @@
 package com.diagbot.config.redislimiter;
 
+import com.diagbot.util.StringUtil;
 import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.cloud.gateway.filter.ratelimit.AbstractRateLimiter;
 import org.springframework.cloud.gateway.filter.ratelimit.RateLimiter;
 import org.springframework.cloud.gateway.support.ConfigurationService;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
 import org.springframework.data.redis.core.ReactiveRedisTemplate;
+import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.data.redis.core.script.RedisScript;
 import org.springframework.util.ObjectUtils;
 import org.springframework.validation.annotation.Validated;
@@ -49,6 +53,10 @@ public class SystemRedisRateLimiter extends AbstractRateLimiter<SystemRedisRateL
     private RedisScript<List<Long>> script;
     private AtomicBoolean initialized = new AtomicBoolean(false);
 
+    @Autowired
+    @Qualifier("redisTemplateForCache")
+    private RedisTemplate myredisTemplate;
+
     private String remainingHeader = REMAINING_HEADER;
 
     /**
@@ -87,6 +95,15 @@ public class SystemRedisRateLimiter extends AbstractRateLimiter<SystemRedisRateL
         if (ObjectUtils.isEmpty(rateLimiterConf)) {
             throw new IllegalArgumentException("No Configuration found for route " + routeId);
         }
+        //黑名单校验
+        String blackListKey = getBlackListKey(id);
+        Integer blackListValue = (Integer)this.myredisTemplate.opsForValue().get(blackListKey);
+        if (null != blackListValue) {
+            if (blackListValue > 20) {
+                throw new IllegalArgumentException("In Blacklist: " + blackListKey);
+            }
+        }
+
         Map<String, Integer> rateLimitMap = rateLimiterConf.getRateLimitMap();
         //缓存的key
         String replenishRateKey = routeId + "." + REPLENISH_RATE_KEY;
@@ -112,6 +129,10 @@ public class SystemRedisRateLimiter extends AbstractRateLimiter<SystemRedisRateL
                         boolean allowed = results.get(0) == 1L;
                         Long tokensLeft = results.get(1);
 
+                        if (!allowed) {
+                            this.myredisTemplate.opsForValue().increment(blackListKey, 1);
+                        }
+
                         RateLimiter.Response response = new RateLimiter.Response(allowed, getHeaders(replenishRate, burstCapacity, tokensLeft));
 
                         return response;
@@ -151,6 +172,17 @@ public class SystemRedisRateLimiter extends AbstractRateLimiter<SystemRedisRateL
         return Arrays.asList(tokenKey, timestampKey);
     }
 
+    static String getBlackListKey(String id) {
+        String ip = id.substring(0, id.indexOf("_"));
+        String key = "ip_black_list_" + ip;
+        return key;
+    }
+
+    static String getBlackListKey(String routeId, String id) {
+        String ip = id.substring(0, id.indexOf("_"));
+        String key = "ip_black_list_" + routeId + "_" + ip;
+        return key + "_" + ip;
+    }
 
     @Validated
     public static class Config {