... | @@ -46,6 +46,7 @@ equity_penetration_qcc |
... | @@ -46,6 +46,7 @@ equity_penetration_qcc |
|
equity_penetration_qcc_login
|
|
equity_penetration_qcc_login
|
|
equity_penetration_qcc_test
|
|
equity_penetration_qcc_test
|
|
equity_penetration_tyc_test
|
|
equity_penetration_tyc_test
|
|
|
|
equity_penetration_tyc
|
|
```
|
|
```
|
|
|
|
|
|
## 采集网站(采集入口)
|
|
## 采集网站(采集入口)
|
... | @@ -54,10 +55,12 @@ equity_penetration_tyc_test |
... | @@ -54,10 +55,12 @@ equity_penetration_tyc_test |
|
```buildoutcfg
|
|
```buildoutcfg
|
|
官网PC端入口:
|
|
官网PC端入口:
|
|
https://www.qcc.com
|
|
https://www.qcc.com
|
|
|
|
https://www.tianyancha.com
|
|
|
|
|
|
采集文件存放路径:
|
|
采集文件存放路径:
|
|
/data/gravel_spiders/equity_penetration_qcc
|
|
/data/gravel_spiders/equity_penetration_qcc
|
|
/data/gravel_spiders/equity_penetration_qcc_login
|
|
/data/gravel_spiders/equity_penetration_qcc_login
|
|
|
|
/data/gravel_spiders/equity_penetration_tyc
|
|
```
|
|
```
|
|
+ 对外测试
|
|
+ 对外测试
|
|
```buildoutcfg
|
|
```buildoutcfg
|
... | @@ -101,6 +104,7 @@ equity_penetration_qcc |
... | @@ -101,6 +104,7 @@ equity_penetration_qcc |
|
equity_penetration_qcc_login (登录)
|
|
equity_penetration_qcc_login (登录)
|
|
equity_penetration_qcc_test (对外测试)
|
|
equity_penetration_qcc_test (对外测试)
|
|
equity_penetration_tyc_test (对外测试)
|
|
equity_penetration_tyc_test (对外测试)
|
|
|
|
equity_penetration_tyc
|
|
```
|
|
```
|
|
|
|
|
|
<!--spider_name-->
|
|
<!--spider_name-->
|
... | @@ -121,6 +125,7 @@ equity_penetration_tyc_test (对外测试) |
... | @@ -121,6 +125,7 @@ equity_penetration_tyc_test (对外测试) |
|
* qcc_login
|
|
* qcc_login
|
|
* qcc_test
|
|
* qcc_test
|
|
* tyc_test
|
|
* tyc_test
|
|
|
|
* tyc
|
|
|
|
|
|
### 优先级队列说明
|
|
### 优先级队列说明
|
|
* equity_penetration 支持队列优先级
|
|
* equity_penetration 支持队列优先级
|
... | @@ -248,10 +253,15 @@ scrapy |
... | @@ -248,10 +253,15 @@ scrapy |
|
|
|
|
|
## 爬虫部署信息
|
|
## 爬虫部署信息
|
|
<!--部署在哪些机器?每个机器多少进程?项目名称是什么?-->
|
|
<!--部署在哪些机器?每个机器多少进程?项目名称是什么?-->
|
|
```buildoutcfg
|
|
```yaml
|
|
target: node_43,node_42,node_32,node_33,node_29,node_28
|
|
target: node_43,node_42,node_32,node_33,node_29,node_28
|
|
project: equity_penetration
|
|
project: equity_penetration
|
|
spider_name: equity_penetration_qcc,equity_penetration_qcc_login,equity_penetration_qcc_test,equity_penetration_tyc_test
|
|
spider_name:
|
|
|
|
- equity_penetration_qcc,
|
|
|
|
- equity_penetration_qcc_login
|
|
|
|
- equity_penetration_qcc_test
|
|
|
|
- equity_penetration_tyc_test
|
|
|
|
- equity_penetration_tyc
|
|
```
|
|
```
|
|
|
|
|
|
### 常规部署
|
|
### 常规部署
|
... | @@ -333,6 +343,7 @@ task_result=8000 # 参数错误 |
... | @@ -333,6 +343,7 @@ task_result=8000 # 参数错误 |
|
/data/gravel_spiders/equity_penetration_qcc
|
|
/data/gravel_spiders/equity_penetration_qcc
|
|
/data/gravel_spiders/equity_penetration_qcc_login
|
|
/data/gravel_spiders/equity_penetration_qcc_login
|
|
/data/gravel_spiders/equity_penetration_qcc_test
|
|
/data/gravel_spiders/equity_penetration_qcc_test
|
|
|
|
/data/gravel_spiders/equity_penetration_tyc
|
|
/data/gravel_spiders/equity_penetration_tyc_test
|
|
/data/gravel_spiders/equity_penetration_tyc_test
|
|
```
|
|
```
|
|
|
|
|
... | | ... | |