|
|
|
# title1
|
|
|
|
|
|
|
|
```puml
|
|
|
|
@startuml
|
|
|
|
|
|
|
|
file ic_crawler_bson_10.8.6.227 as bson_aliyun
|
|
|
|
file ic_crawler_bson_192.168.109.45 as bson_office
|
|
|
|
queue table_kafka
|
|
|
|
database ic
|
|
|
|
file update_data_json
|
|
|
|
queue table_update_redis
|
|
|
|
file update_data_json_province
|
|
|
|
|
|
|
|
bson_aliyun --> bson_office
|
|
|
|
bson_office --> ic: 小表,bson_reader,sync_mysql_writer
|
|
|
|
bson_office --> table_kafka: 大表,bson_reader,kafka_writer
|
|
|
|
table_kafka --> ic: kafka_reader,sync_mysql_filter,redis_writer
|
|
|
|
ic --> update_data_json: 根据update_time导出增量,sql_reader,file_writer
|
|
|
|
ic --> table_update_redis: 入库时将变化记录的主键写入redis
|
|
|
|
table_update_redis --> update_data_json: redis_reader,udm_filter,file_writer
|
|
|
|
update_data_json --> update_data_json_province: 按省份分目录
|
|
|
|
@enduml
|
|
|
|
```
|
|
|
|
|
|
|
|
# 爬虫结果同步到mysql库
|
|
|
|
|
|
|
|
## 更新量少的表直接入库
|
|
|
|
|
|
|
|
project-collie-app/app_lake/data_pump/sync_online/ic_spider_update_lake.yml
|
|
|
|
|
|
|
|
## 更新量大的表写kafka队列然后入库
|
|
|
|
|
|
|
|
写kafka队列:
|
|
|
|
|
|
|
|
project-collie-app/app_lake/data_pump/sync_online/ic_spider_update_lake.yml
|
|
|
|
|
|
|
|
读kafka更新mysql并写redis:
|
|
|
|
|
|
|
|
project-collie-app/app_lake/data_pump/ic_lake/collie_ic_spider_update_lake_kafka.yml
|
|
|
|
|
|
|
|
|
|
|
|
| data_type | topic | group |
|
|
|
|
| ------ | ------ | ------ |
|
|
|
|
| ic_base | ic-spider-ic_base | ic_spider_base-7f25e12efd197862ba57ee1ac4957092 |
|
|
|
|
| ic_branch | ic-spider-ic_branch | ic_spider_branch-0dd285a5745e367b426e0d39339e6320 |
|
|
|
|
| ic_change | ic-spider-ic_change | ic_spider_change-9faa47b4eab8c0e72c7a01bcdcfd90f3 |
|
|
|
|
| ic_employee | ic-spider-ic_employee | ic_spider_employee-ca5b26304c5f784f5c19967e708e2681 |
|
|
|
|
| ic_history_name | ic-spider-ic_history_name | ic_spider_history_name-6bf263431bb684568ef885902343f186 |
|
|
|
|
| ic_industry | ic-spider-ic_industry | ic_spider_industry-3e5fa1404295a2ef7e692866b23c28e4 |
|
|
|
|
| ic_partner | ic-spider-ic_partner | ic_spider_partner-f5ece1f5cccbf44179b3b5f59f76c030 |
|
|
|
|
| ar_base | ic-spider-ar_base | ic_spider_ar_base-7c60cedcf9e3712602c0fc9c78d20c07 |
|
|
|
|
| ar_partner | ic-spider-ar_partner | ic_spider_ar_partner-12d5ce6dd15e7e966659ed386195e558 |
|
|
|
|
|
|
|
|
|
|
|
|
# 增量数据从mysql库中导出
|
|
|
|
|
|
|
|
## 增量数据根据update_time从mysql获取
|
|
|
|
|
|
|
|
project-collie-app/app_lake/data_pump/lake/lake_ic.yml
|
|
|
|
|
|
|
|
## 增量数据读redis查mysql获取
|
|
|
|
|
|
|
|
project-collie-app/app_lake/data_pump/ic_lake/collie_ic_spider_update_lake_redis.yml |
|
|
\ No newline at end of file |
