| ... | ... | @@ -90,7 +90,6 @@ queue "topic: collie-ic-data-digest" |
|
|
|
queue "topic: ic-data-clean-result"
|
|
|
|
queue "topic: ic_ods_binlog"
|
|
|
|
|
|
|
|
database "es: ic_spider_task"
|
|
|
|
|
|
|
|
[spider_log] --> [topic: collie-ic-data-to-mongo]: logstash
|
|
|
|
|
| ... | ... | @@ -100,13 +99,13 @@ database "es: ic_spider_task" |
|
|
|
[topic: collie-ic-data-digest] --> [udms: ic_spider_data]: 多进程清洗
|
|
|
|
[udms: ic_spider_data] --> [topic: ic-data-clean-result]
|
|
|
|
[topic: ic-data-clean-result] --> [udm: sync_mysql_new]: 多进程入库
|
|
|
|
[topic: ic-data-clean-result] --> [es: ic_spider_task]: 更新爬虫任务
|
|
|
|
[es: ic_spider_task] --> [spider_log]
|
|
|
|
[udm: sync_mysql_new] --> [company_base]
|
|
|
|
[udm: sync_mysql_new] --> [company_legalperson]
|
|
|
|
[udm: sync_mysql_new] --> [company_partner]
|
|
|
|
[udm: sync_mysql_new] --> [company_...]
|
|
|
|
|
|
|
|
[company_base] --> [spider_log]: 爬虫滚动更新任务
|
|
|
|
|
|
|
|
[company_base] --> [topic: ic_ods_binlog]: 一表一分区
|
|
|
|
[company_legalperson] --> [topic: ic_ods_binlog]: 一表一分区
|
|
|
|
[company_partner] --> [topic: ic_ods_binlog]: 一表一分区
|
| ... | ... | |
