... | @@ -332,27 +332,31 @@ scrapy |
... | @@ -332,27 +332,31 @@ scrapy |
|
## 归集后存放目录
|
|
## 归集后存放目录
|
|
```
|
|
```
|
|
/data2_227/grvael_spider_result/law_office
|
|
/data2_227/grvael_spider_result/law_office
|
|
|
|
/data2_227/grvael_spider_result/bj_law_office
|
|
|
|
/data2_227/grvael_spider_result/sh_law_office
|
|
```
|
|
```
|
|
|
|
|
|
## logstash配置文件名称
|
|
## logstash配置文件名称
|
|
```
|
|
```
|
|
project-deploy/logstash/10.8.6.246/conf.d/collie_spider_data_to_kfk.conf(入topic)
|
|
project-deploy/logstash/10.8.6.246/conf.d/collie_spider_data_to_kfk.conf(入topic)
|
|
project-deploy/logstash/10.8.6.229/conf.d/grvael_spider_to_es.conf(入es)
|
|
|
|
```
|
|
```
|
|
|
|
|
|
## logstash文件采集type
|
|
## logstash文件采集type
|
|
```
|
|
```
|
|
type=>"law_office"
|
|
type=>"law_office"
|
|
|
|
type=>"bj_law_office"
|
|
|
|
type=>"sh_law_office"
|
|
```
|
|
```
|
|
|
|
|
|
## 数据归集的topic
|
|
## 数据归集的topic
|
|
```
|
|
```
|
|
topic_id => "public-company-spider-data"
|
|
topic_id => "general-taxpayer"
|
|
|
|
分区数: 12
|
|
|
|
数据清洗group => "law_office_etl"
|
|
```
|
|
```
|
|
|
|
|
|
## ES日志索引及筛选条件
|
|
## ES日志索引及筛选条件
|
|
```
|
|
```
|
|
index => "public-company-spider-data-%{log_date}"
|
|
|
|
```
|
|
```
|
|
|
|
|
|
## 监控指标看板
|
|
## 监控指标看板
|
... | | ... | |