... | ... | @@ -25,7 +25,7 @@ http://12348.moj.gov.cn/#/publicies/lawdept/lawdept |
|
|
## 采集频率及采集策略
|
|
|
<!--说明该项数据更新频率,存量数据的更新频率及策略、新增数据的采集频率及策略-->
|
|
|
```
|
|
|
目前暂定每天更新一遍
|
|
|
基本是一次性工作,暂不需要周期调度
|
|
|
```
|
|
|
|
|
|
### 存量更新策略
|
... | ... | @@ -58,10 +58,14 @@ http://12348.moj.gov.cn/#/publicies/lawdept/lawdept |
|
|
|
|
|
## 代码地址
|
|
|
```
|
|
|
项目地址:http://192.168.109.110/granite/project-gravel/tree/develop_law_office_20210819/scrapy_spiders/gravel_spiders/spiders
|
|
|
项目地址
|
|
|
http://192.168.109.110/granite/project-gravel/-/tree/develop_adsearch_20210517/scrapy_spiders/gravel_spiders/spiders
|
|
|
|
|
|
项目入口脚本: http://192.168.109.110/granite/project-gravel/tree/develop_law_office_20210819/scrapy_spiders/gravel_spiders/spiders/law_office.py
|
|
|
代码具体实现板块: http://192.168.109.110/granite/project-gravel/tree/develop_law_office_20210819/scrapy_spiders/gravel_spiders/spiders/law_office_reqs
|
|
|
项目入口脚本
|
|
|
http://192.168.109.110/granite/project-gravel/-/blob/develop_adsearch_20210517/scrapy_spiders/gravel_spiders/spiders/law_office.py
|
|
|
|
|
|
代码具体实现板块
|
|
|
http://192.168.109.110/granite/project-gravel/-/tree/develop_adsearch_20210517/scrapy_spiders/gravel_spiders/spiders/law_office_reqs
|
|
|
```
|
|
|
|
|
|
## 队列名称及队列地址
|
... | ... | @@ -103,17 +107,16 @@ http://12348.moj.gov.cn/lawerdeptlist/getlawerdeptlist |
|
|
"platform": "law_office",
|
|
|
"data_type: "list",
|
|
|
}
|
|
|
说明: 任务直接到redis,不经过taskhub
|
|
|
```
|
|
|
|
|
|
### 任务参数说明
|
|
|
<!--特有参数说明,通用参数比如spider_name,task_params,task_src,task_result等不需说明-->
|
|
|
```json
|
|
|
{
|
|
|
"platform": "law_office",
|
|
|
"area_code": "law_office",
|
|
|
"data_type": "list"
|
|
|
}
|
|
|
|
|
|
说明: 任务直接到redis,不经过taskhub
|
|
|
```
|
|
|
|
|
|
## data_type说明
|
... | ... | @@ -121,38 +124,44 @@ http://12348.moj.gov.cn/lawerdeptlist/getlawerdeptlist |
|
|
```
|
|
|
list: 入口url
|
|
|
detail: 某个详情页的数据
|
|
|
log: 日志记录
|
|
|
```
|
|
|
|
|
|
## 爬虫结果的超级数据
|
|
|
<!--包含所有字段的json数据,每个value都要有样例值-->
|
|
|
```
|
|
|
{
|
|
|
"data": {
|
|
|
"law_office": {
|
|
|
"name": "辽宁伯宁(北京)律师事务所",
|
|
|
"credit_no": "31110000359004104P",
|
|
|
"address": "北京市东城区建国门内大街7号6层12室",
|
|
|
"establish_date": "2015-10-13",
|
|
|
"lawyer_num": "3",
|
|
|
"area": "东城区",
|
|
|
"phone_number": "65186566",
|
|
|
"desc": "辽宁伯宁律师事务所创建于2009年9月,总部设在大连,是一家专长于海商海事、国际仲裁、航运金融、融资租赁、建筑工程、公司商事、房地产、建筑施工、投融资、知识产权、劳动争议、民事诉讼以及刑事辩护等领域的全方位、综合性律师事务所。"
|
|
|
},
|
|
|
"data": [{
|
|
|
"jj": "",
|
|
|
"pzslsj": "2010-05-13",
|
|
|
"zzxs": "02",
|
|
|
"lsswsmc": "重庆道润律师事务所",
|
|
|
"zsdh": "023-67686602",
|
|
|
"zyzh": "25001201010281592",
|
|
|
"cz": "023-67686602",
|
|
|
"zsd": "重庆市北部新区高新园区宝华大道10号1幢1区17-6、17-7、17-8、17-9、17-10",
|
|
|
"jd": "106.55640000004154",
|
|
|
"tyshxydm": "31500000552049937N",
|
|
|
"wd": "29.610119006729243",
|
|
|
"ywzclist": [],
|
|
|
"nums": 5
|
|
|
}],
|
|
|
"http_code": 200,
|
|
|
"error_msg": "",
|
|
|
"task_result": 1000,
|
|
|
"data_type": "detail",
|
|
|
"spider_start_time": "2021-06-18 11:39:51.724",
|
|
|
"spider_end_time": "2021-06-18 11:39:53.034",
|
|
|
"spider_start_time": "2021-08-20 13:14:27.338",
|
|
|
"spider_end_time": "2021-08-20 13:16:30.341",
|
|
|
"task_params": {
|
|
|
"platform": "law_office",
|
|
|
"data_type": "list"
|
|
|
},
|
|
|
"metadata": {
|
|
|
"current_page": 2
|
|
|
"current_page": 2515
|
|
|
},
|
|
|
"spider_name": "law_office",
|
|
|
"spider_ip": "10.8.1.38"
|
|
|
"spider_ip": "10.8.1.10",
|
|
|
"proxy_ip": "http://127.0.0.1:8888"
|
|
|
}
|
|
|
|
|
|
```
|
... | ... | |