| ... | ... | @@ -4,13 +4,15 @@ |
|
|
|
执行公开爬虫(包含：'zhixing','shixin','xianxiao','zhongben')
|
|
|
|
zhixing: 通过增加的id和数据库已有id进行找新和更新,如果没有id，通过pname查找
|
|
|
|
shixin: 通过增加的id进行找新，更新
|
|
|
|
zhongben: 通过增加的id进行找新，更新
|
|
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
## 数据名称（中文）
|
|
|
|
|
|
|
|
<!-- 该项数据的中文标准名称，后续沟通交流使用的规范名称，如：工商公示股东信息、失信被执行人、一般纳税人等-->
|
|
|
|
|
|
|
|
###
|
|
|
|
###
|
|
|
|
|
|
|
|
```buildoutcfg
|
|
|
|
执行公开信息
|
| ... | ... | @@ -72,7 +74,7 @@ db_password: |
|
|
|
* 根据表里面最大zhixing_id每天生成十万任务进行更新
|
|
|
|
|
|
|
|
* 终本案件
|
|
|
|
* 待补充
|
|
|
|
* 根据表中zhongben_id的最大值每天生成十万任务
|
|
|
|
|
|
|
|
* 限制高消费
|
|
|
|
* 待补充
|
| ... | ... | @@ -145,28 +147,28 @@ risk_zxgk |
|
|
|
```json
|
|
|
|
通用任务
|
|
|
|
zhixing
|
|
|
|
{
|
|
|
|
"zhixing_id": "1371041114",
|
|
|
|
"task_type": "spider_zhixing_task"
|
|
|
|
}
|
|
|
|
{
|
|
|
|
"zhixing_id": "1371041114",
|
|
|
|
"task_type": "spider_zhixing_task"
|
|
|
|
}
|
|
|
|
|
|
|
|
shixin
|
|
|
|
{
|
|
|
|
"shixin_id": "1371041114",
|
|
|
|
"task_type": "spider_shixin_task"
|
|
|
|
}
|
|
|
|
{
|
|
|
|
"shixin_id": "1371041114",
|
|
|
|
"task_type": "spider_shixin_task"
|
|
|
|
}
|
|
|
|
|
|
|
|
xianxiao
|
|
|
|
{
|
|
|
|
"xianxiao_id": "1371041114",
|
|
|
|
"task_type": "spider_xianxiao_task"
|
|
|
|
}
|
|
|
|
{
|
|
|
|
"xianxiao_id": "1371041114",
|
|
|
|
"task_type": "spider_xianxiao_task"
|
|
|
|
}
|
|
|
|
|
|
|
|
zhongben
|
|
|
|
{
|
|
|
|
"zhongben_id": "1371041114",
|
|
|
|
"task_type": "spider_zhongben_task"
|
|
|
|
}
|
|
|
|
{
|
|
|
|
"zhongben_id": "1371041114",
|
|
|
|
"task_type": "spider_zhongben_task"
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
zhixing 通过pname查询
|
| ... | ... | @@ -179,11 +181,14 @@ zhixing 通过pname查询 |
|
|
|
```
|
|
|
|
|
|
|
|
### 任务样例
|
|
|
|
|
|
|
|
<!--注意是爬虫拿到的完整任务，而不仅仅是task_params-->
|
|
|
|
|
|
|
|
```buildoutcfg
|
|
|
|
```
|
|
|
|
|
|
|
|
### 任务参数说明
|
|
|
|
|
|
|
|
<!--特有参数说明，通用参数比如spider_name,task_params,task_src,task_result等不需说明-->
|
|
|
|
|
|
|
|
```json
|
| ... | ... | @@ -197,6 +202,7 @@ pname_id_task: 这个任务的类型，是否是通过pname查询 |
|
|
|
```
|
|
|
|
|
|
|
|
## data_type说明
|
|
|
|
|
|
|
|
<!--可能产生的data_type说明-->
|
|
|
|
|
|
|
|
```buildoutcfg
|
| ... | ... | @@ -207,6 +213,7 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
## 爬虫结果的超级数据
|
|
|
|
|
|
|
|
<!--包含所有字段的json数据，每个value都要有样例值-->
|
|
|
|
|
|
|
|
```json
|
|
|
|
{
|
|
|
|
"data": [
|
| ... | ... | @@ -221,7 +228,34 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
"cardnum_view": "32048219981****2812",
|
|
|
|
"sex_name": "男性",
|
|
|
|
"spider_time": "2021-07-06 19:40:09",
|
|
|
|
"deleted": 0
|
|
|
|
"deleted": 0,
|
|
|
|
"id": 713347040,
|
|
|
|
"iname": "翟振海",
|
|
|
|
"caseCode": "（2021）桂0981执400号",
|
|
|
|
"age": 57,
|
|
|
|
"sexy": "男性",
|
|
|
|
"cardNum": "3421291962****761X",
|
|
|
|
"courtName": "北流市人民法院",
|
|
|
|
"areaName": "广西",
|
|
|
|
"partyTypeName": "0",
|
|
|
|
"gistId": "（2020）桂0981刑初147号",
|
|
|
|
"regDate": "2021年01月14日",
|
|
|
|
"gistUnit": "北流市人民法院",
|
|
|
|
"duty": "（2020）桂0981刑初147号",
|
|
|
|
"performance": "全部未履行",
|
|
|
|
"performedPart": "暂无",
|
|
|
|
"unperformPart": "暂无",
|
|
|
|
"disruptTypeName": "违反财产报告制度",
|
|
|
|
"publishDate": "2021年07月06日",
|
|
|
|
"case_code": "2006年高执字第00497号",
|
|
|
|
"cysah": "（2005）民三终字第5号",
|
|
|
|
"zhongben_id": 2,
|
|
|
|
"name": "北京嘉裕东方葡萄酒有限公司",
|
|
|
|
"zhongben_date": "2007年12月20日",
|
|
|
|
"lian_date": "2006年10月16日",
|
|
|
|
"cardnum_view": "72261909-1",
|
|
|
|
"exec_money": 7474752,
|
|
|
|
"court_name": "北京市高级人民法院"
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"http_code": 200,
|
| ... | ... | @@ -232,9 +266,7 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
"spider_end_time": "2021-07-06 19:40:09",
|
|
|
|
"task_params": {
|
|
|
|
"zhixing_id": "1370997737",
|
|
|
|
"task_type": "spider_zhixing_task",
|
|
|
|
"pname": "徐琰炜",
|
|
|
|
"pname_id_task": false
|
|
|
|
"task_type": "spider_zhixing_task"
|
|
|
|
},
|
|
|
|
"metadata": {},
|
|
|
|
"spider_name": "risk_zxgk",
|
| ... | ... | @@ -243,10 +275,12 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
```
|
|
|
|
|
|
|
|
## 实际爬虫结果的数据结构
|
|
|
|
|
|
|
|
<!--可能与超级数据一致，可能不同的data_type的爬虫结果结构不同，超级数据是把所有data_type的结果组合在一起-->
|
|
|
|
|
|
|
|
* zhixing
|
|
|
|
* 通过zhingxing_id爬取
|
|
|
|
* 通过zhingxing_id爬取
|
|
|
|
|
|
|
|
```json
|
|
|
|
{
|
|
|
|
"data": [
|
| ... | ... | @@ -261,25 +295,7 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
"cardnum_view": "32048219981****2812",
|
|
|
|
"sex_name": "男性",
|
|
|
|
"spider_time": "2021-07-06 19:40:09",
|
|
|
|
"deleted": 0,
|
|
|
|
"id": 713347040,
|
|
|
|
"iname": "翟振海",
|
|
|
|
"caseCode": "（2021）桂0981执400号",
|
|
|
|
"age": 57,
|
|
|
|
"sexy": "男性",
|
|
|
|
"cardNum": "3421291962****761X",
|
|
|
|
"courtName": "北流市人民法院",
|
|
|
|
"areaName": "广西",
|
|
|
|
"partyTypeName": "0",
|
|
|
|
"gistId": "（2020）桂0981刑初147号",
|
|
|
|
"regDate": "2021年01月14日",
|
|
|
|
"gistUnit": "北流市人民法院",
|
|
|
|
"duty": "（2020）桂0981刑初147号",
|
|
|
|
"performance": "全部未履行",
|
|
|
|
"performedPart": "暂无",
|
|
|
|
"unperformPart": "暂无",
|
|
|
|
"disruptTypeName": "违反财产报告制度",
|
|
|
|
"publishDate": "2021年07月06日"
|
|
|
|
"deleted": 0
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"http_code": 200,
|
| ... | ... | @@ -290,14 +306,18 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
"spider_end_time": "2021-07-06 19:40:09",
|
|
|
|
"task_params": {
|
|
|
|
"zhixing_id": "1370997737",
|
|
|
|
"task_type": "spider_zhixing_task"
|
|
|
|
"task_type": "spider_zhixing_task",
|
|
|
|
"pname": "徐琰炜",
|
|
|
|
"pname_id_task": false
|
|
|
|
},
|
|
|
|
"metadata": {},
|
|
|
|
"spider_name": "risk_zxgk",
|
|
|
|
"spider_ip": "10.8.6.51"
|
|
|
|
}
|
|
|
|
|
|
|
|
```
|
|
|
|
* 通过pname爬取
|
|
|
|
|
|
|
|
* 通过pname爬取
|
|
|
|
|
|
|
|
```json
|
|
|
|
{
|
| ... | ... | @@ -336,6 +356,7 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
```
|
|
|
|
|
|
|
|
* shixin
|
|
|
|
|
|
|
|
```json
|
|
|
|
{
|
|
|
|
"data": [
|
| ... | ... | @@ -377,17 +398,46 @@ detail: 详情信息 列表,其中每个字典为一条数据,只有这一种类 |
|
|
|
```
|
|
|
|
|
|
|
|
* xianxiao
|
|
|
|
|
|
|
|
```json
|
|
|
|
待补充
|
|
|
|
```
|
|
|
|
|
|
|
|
* zhongben
|
|
|
|
|
|
|
|
```json
|
|
|
|
待补充
|
|
|
|
{
|
|
|
|
"data": [
|
|
|
|
{
|
|
|
|
"case_code": "2006年高执字第00497号",
|
|
|
|
"cysah": "（2005）民三终字第5号",
|
|
|
|
"zhongben_id": 2,
|
|
|
|
"name": "北京嘉裕东方葡萄酒有限公司",
|
|
|
|
"zhongben_date": "2007年12月20日",
|
|
|
|
"lian_date": "2006年10月16日",
|
|
|
|
"cardnum_view": "72261909-1",
|
|
|
|
"exec_money": 7474752,
|
|
|
|
"court_name": "北京市高级人民法院"
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"http_code": 200,
|
|
|
|
"error_msg": "",
|
|
|
|
"task_result": 1000,
|
|
|
|
"data_type": "detail",
|
|
|
|
"spider_start_time": "2021-07-12 17:08:04.833",
|
|
|
|
"spider_end_time": "2021-07-12 17:08:06",
|
|
|
|
"task_params": {
|
|
|
|
"zhongben_id": "2",
|
|
|
|
"task_type": "spider_zhongben_task"
|
|
|
|
},
|
|
|
|
"metadata": {},
|
|
|
|
"spider_name": "risk_zxgk",
|
|
|
|
"spider_ip": "192.168.56.1"
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
## 爬虫运行环境
|
|
|
|
|
|
|
|
<!--udm模块？scrapy？或其他-->
|
|
|
|
|
|
|
|
```buildoutcfg
|
| ... | ... | @@ -395,6 +445,7 @@ scrapy |
|
|
|
```
|
|
|
|
|
|
|
|
## 爬虫部署信息
|
|
|
|
|
|
|
|
<!--部署在哪些机器？每个机器多少进程？项目名称是什么？-->
|
|
|
|
|
|
|
|
```buildoutcfg
|
| ... | ... | @@ -419,6 +470,7 @@ task_result=8000 # 参数错误 |
|
|
|
```
|
|
|
|
|
|
|
|
## 爬虫监控指标设计
|
|
|
|
|
|
|
|
<!--监控爬虫正常运行的指标是什么？报警规则是什么？-->
|
|
|
|
|
|
|
|
```buildoutcfg
|
| ... | ... | |
